request timeout, curl in NAT

This commit is contained in:
2021-08-12 23:13:49 -06:00
parent 6bfa807ac4
commit c44122ec8c
3 changed files with 24 additions and 12 deletions

View File

@@ -13,13 +13,15 @@
(readComponent archive e Name)
:Array<String> wikipediaImageUrls
(queryImageUrls (queryImageTitles [title]))]
(doFor url wikipediaImageUrls
(assertProcess "wget" ["--directory-prefix=${archive.archiveDir}" url]))
(addFiles archive e
(for url wikipediaImageUrls
(joinPath archive.archiveDir
~(.replace
(.urlDecode (url.withoutDirectory))
// Some symbols shouldn't be decoded because they're invalid in file systems!
"\"" "%22"))))
(let [filePath
(joinPath archive.archiveDir
~(.replace
(.urlDecode (url.withoutDirectory))
// Some symbols shouldn't be decoded because they're invalid in file systems!
"\"" "%22"))]
(print url "downloading")
(assertProcess "curl" ["--output" filePath url])
filePath)))
(addTags archive e ["wikipediaProcessed"])))