vbatts/fetch-content
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

main.go: ruling out a colliding metadata for single PDF

Browse source 
Prompt claude.ai using 3.7 Sonnet:
```
one additional issue that the metadata json file is also named as the
hash of the PDF, but if there are two different URLs that download a
matching PDF, then while the hash will be the same the metadata is
actually different for the two, so it will collide there.  Lets also get
a sha1 hash of the URL string itself, and the metadata json file is
named with hash of the PDF, then a hyphen, then hash of the URL.
```

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
This commit is contained in:
Vincent Batts 2025-04-03 22:26:40 -04:00
parent 4400ea5bca
commit 67e835eea9
Signed by: vbatts
GPG key ID: E30EFAA812C6E5ED
1 changed files with 43 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
main.go
View file

@ -22,6 +22,7 @@ import (
// FileMetadata stores information about a downloaded file
type FileMetadata struct {
URL string `json:"url"`
URLHash string `json:"url_hash"`
ContentDisposition string `json:"content_disposition,omitempty"`
SuggestedFilename string `json:"suggested_filename,omitempty"`
ActualFilename string `json:"actual_filename"`
@ -45,6 +46,13 @@ type Config struct {
Verbose bool
}
// Calculate SHA-1 hash of a string
func hashString(s string) string {
h := sha1.New()
h.Write([]byte(s))
return fmt.Sprintf("%x", h.Sum(nil))
}
func main() {
// Parse command line flags
concurrency := flag.Int("concurrency", 10, "Number of concurrent downloads")
@ -204,6 +212,9 @@ func downloadWithRetry(ctx context.Context, client *http.Client, url string, con
}
func downloadURL(ctx context.Context, client *http.Client, url string, config Config, workerID int) error {
// Calculate URL hash upfront
urlHash := hashString(url)
// Create HTTP request with context for cancellation
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
@ -275,34 +286,50 @@ func downloadURL(ctx context.Context, client *http.Client, url string, config Co
}
// Get the SHA-1 checksum
sha1sum := fmt.Sprintf("%x", hash.Sum(nil))
contentHash := fmt.Sprintf("%x", hash.Sum(nil))
// Check if we already have this file
if config.SkipExisting {
existingPath, exists := findExistingFile(config.MetadataDir, sha1sum)
if exists {
// Check if this exact URL and content combination has already been downloaded
metadataFilename := fmt.Sprintf("%s-%s.json", contentHash, urlHash)
metadataPath := filepath.Join(config.MetadataDir, metadataFilename)
if _, err := os.Stat(metadataPath); err == nil && config.SkipExisting {
if config.Verbose {
log.Printf("File with SHA-1 %s already exists at %s, skipping", sha1sum, existingPath)
log.Printf("This URL and content combination already exists, skipping: %s", url)
}
return nil
}
// Check if we already have this file content from a different URL
existingFile := ""
if config.SkipExisting {
// Look for any file with this content hash
contentFiles, err := filepath.Glob(filepath.Join(config.OutputDir, contentHash+"*"))
if err == nil && len(contentFiles) > 0 {
existingFile = contentFiles[0]
if config.Verbose {
log.Printf("File with content hash %s already exists at %s, reusing", contentHash, existingFile)
}
}
}
// Create the target filename based on SHA-1
targetFilename := sha1sum
targetFilename := contentHash
if filepath.Ext(suggestedFilename) != "" {
// Append original extension if available
targetFilename = fmt.Sprintf("%s%s", sha1sum, filepath.Ext(suggestedFilename))
targetFilename = fmt.Sprintf("%s%s", contentHash, filepath.Ext(suggestedFilename))
}
targetPath := filepath.Join(config.OutputDir, targetFilename)
// Close the temp file before copying its contents
tempFile.Close()
// Copy the file if it doesn't already exist
if existingFile == "" {
// Fix: Instead of using os.Rename, copy the file contents to handle cross-partition moves
if err := copyFileAcrossPartitions(tempFilePath, targetPath); err != nil {
return fmt.Errorf("failed to copy file: %w", err)
}
}
// Parse Last-Modified header
var lastModified time.Time
@ -314,10 +341,11 @@ func downloadURL(ctx context.Context, client *http.Client, url string, config Co
// Create metadata
metadata := FileMetadata{
URL: url,
URLHash: urlHash,
ContentDisposition: contentDisposition,
SuggestedFilename: suggestedFilename,
ActualFilename: targetFilename,
SHA1Checksum: sha1sum,
SHA1Checksum: contentHash,
ETag: resp.Header.Get("ETag"),
LastModified: lastModified,
DownloadedAt: time.Now(),
@ -325,8 +353,7 @@ func downloadURL(ctx context.Context, client *http.Client, url string, config Co
ContentLength: n,
}
// Write metadata to file
metadataPath := filepath.Join(config.MetadataDir, sha1sum+".json")
// Write metadata to file with combined hash name
metadataJSON, err := json.MarshalIndent(metadata, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal metadata: %w", err)
@ -337,7 +364,8 @@ func downloadURL(ctx context.Context, client *http.Client, url string, config Co
}
if config.Verbose {
log.Printf("Successfully downloaded %s (%d bytes) to %s", url, n, targetPath)
log.Printf("Successfully processed %s (%d bytes), content stored at %s, metadata at %s",
url, n, targetPath, metadataPath)
}
return nil
@ -382,25 +410,3 @@ func copyFileAcrossPartitions(srcPath, dstPath string) error {
return nil
}
// findExistingFile checks if a file with the given SHA-1 checksum already exists
func findExistingFile(metadataDir, sha1sum string) (string, bool) {
metadataPath := filepath.Join(metadataDir, sha1sum+".json")
_, err := os.Stat(metadataPath)
if err != nil {
return "", false
}
// Read the metadata to get the actual file path
data, err := os.ReadFile(metadataPath)
if err != nil {
return "", false
}
var metadata FileMetadata
if err := json.Unmarshal(data, &metadata); err != nil {
return "", false
}
return metadata.ActualFilename, true
}