request timeout, curl in NAT

projects/nat-archive-tool/src/nat/systems/MediaWikiSystem.kiss

@@ -20,6 +20,8 @@
                 importHxFile "extern-files/python/import.hx"
                 langProjectFile "extern-files/python/requirements.txt")
             [:Map<String,Array<String>> params _ :String mediaWikiUrl _ :String maxLag (if maxLag (Std.string maxLag) "1") :Map<String,String> headers _]
+            
+            (print "extern query call $params")
             (let [response
                         (Requests.get mediaWikiUrl 
                             (let [innerParams
@@ -30,8 +32,11 @@
                                         ]]
                                 (doFor =>param paramValues params (dictSet innerParams param (paramValues.join "|")))
                                 innerParams)
-                            (object headers headers))]
+                            (object
+                                headers headers
+                                timeout 2))]
                 (assert response.ok)
+                (print "call finished: ${response.text}")
                 response.text))))
 
 (method :Array<String> queryImageTitles [:Array<String> pageTitles]
@@ -43,7 +48,9 @@
 
 (method :Array<String> queryImageUrls [:Array<String> imageTitles]
     (flatten
-        (for =>_id image (the haxe.DynamicAccess<Dynamic> .pages .query (query [=>"titles" imageTitles =>"prop" ["imageinfo"] =>"iiprop" ["url"]]))
+        (for =>_id image
+                (the haxe.DynamicAccess<Dynamic>
+                    .pages .query (query [=>"titles" imageTitles =>"prop" ["imageinfo"] =>"iiprop" ["url"]]))
             (if image.imageinfo
                     (image.imageinfo.map ->image image.url)
                 []))))

projects/nat-archive-tool/src/nat/systems/WikipediaImageSystem.kiss

@@ -13,13 +13,15 @@
                 (readComponent archive e Name)
             :Array<String> wikipediaImageUrls
                 (queryImageUrls (queryImageTitles [title]))]
-        (doFor url wikipediaImageUrls
-            (assertProcess "wget" ["--directory-prefix=${archive.archiveDir}" url]))
         (addFiles archive e
             (for url wikipediaImageUrls
-                (joinPath archive.archiveDir
-                    ~(.replace
-                        (.urlDecode (url.withoutDirectory))
-                        // Some symbols shouldn't be decoded because they're invalid in file systems!
-                        "\"" "%22"))))
+                (let [filePath
+                            (joinPath archive.archiveDir
+                                ~(.replace
+                                    (.urlDecode (url.withoutDirectory))
+                                    // Some symbols shouldn't be decoded because they're invalid in file systems!
+                                    "\"" "%22"))]
+                    (print url "downloading")
+                    (assertProcess "curl" ["--output" filePath url])
+                    filePath)))
         (addTags archive e ["wikipediaProcessed"])))

projects/requests-externs/src/requests_externs/Requests.hx

@@ -5,11 +5,13 @@ import python.Dict;
 import python.KwArgs;
 
 typedef NativeRequestKwArgs = {
-    ?headers:Dict<String, String>
+    ?headers:Dict<String, String>,
+    ?timeout:Float
 }
 
 typedef RequestKwArgs = {
-    ?headers:Map<String, String>
+    ?headers:Map<String, String>,
+    ?timeout:Float
 }
 
 @:pythonImport("requests")
@@ -36,7 +38,8 @@ class Requests {
         if (kwArgs == null)
             return null;
         return {
-            headers: mapToDict(kwArgs.headers)
+            headers: mapToDict(kwArgs.headers),
+            timeout: kwArgs.timeout
         };
     }
 }