diff --git a/src/nat/systems/MediaWikiSystem.kiss b/src/nat/systems/MediaWikiSystem.kiss index 041d03f..d66c4c1 100644 --- a/src/nat/systems/MediaWikiSystem.kiss +++ b/src/nat/systems/MediaWikiSystem.kiss @@ -20,6 +20,8 @@ importHxFile "extern-files/python/import.hx" langProjectFile "extern-files/python/requirements.txt") [:Map> params _ :String mediaWikiUrl _ :String maxLag (if maxLag (Std.string maxLag) "1") :Map headers _] + + (print "extern query call $params") (let [response (Requests.get mediaWikiUrl (let [innerParams @@ -30,8 +32,11 @@ ]] (doFor =>param paramValues params (dictSet innerParams param (paramValues.join "|"))) innerParams) - (object headers headers))] + (object + headers headers + timeout 2))] (assert response.ok) + (print "call finished: ${response.text}") response.text)))) (method :Array queryImageTitles [:Array pageTitles] @@ -43,7 +48,9 @@ (method :Array queryImageUrls [:Array imageTitles] (flatten - (for =>_id image (the haxe.DynamicAccess .pages .query (query [=>"titles" imageTitles =>"prop" ["imageinfo"] =>"iiprop" ["url"]])) + (for =>_id image + (the haxe.DynamicAccess + .pages .query (query [=>"titles" imageTitles =>"prop" ["imageinfo"] =>"iiprop" ["url"]])) (if image.imageinfo (image.imageinfo.map ->image image.url) [])))) diff --git a/src/nat/systems/WikipediaImageSystem.kiss b/src/nat/systems/WikipediaImageSystem.kiss index 4eab282..ff079b8 100644 --- a/src/nat/systems/WikipediaImageSystem.kiss +++ b/src/nat/systems/WikipediaImageSystem.kiss @@ -13,13 +13,15 @@ (readComponent archive e Name) :Array wikipediaImageUrls (queryImageUrls (queryImageTitles [title]))] - (doFor url wikipediaImageUrls - (assertProcess "wget" ["--directory-prefix=${archive.archiveDir}" url])) (addFiles archive e (for url wikipediaImageUrls - (joinPath archive.archiveDir - ~(.replace - (.urlDecode (url.withoutDirectory)) - // Some symbols shouldn't be decoded because they're invalid in file systems! - "\"" "%22")))) + (let [filePath + (joinPath archive.archiveDir + ~(.replace + (.urlDecode (url.withoutDirectory)) + // Some symbols shouldn't be decoded because they're invalid in file systems! + "\"" "%22"))] + (print url "downloading") + (assertProcess "curl" ["--output" filePath url]) + filePath))) (addTags archive e ["wikipediaProcessed"])))