kiss-lang/kiss-vscode
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

request timeout, curl in NAT

Browse Source
This commit is contained in:
Nat Quayle Nelson
2021-08-12 23:13:49 -06:00
parent 6bfa807ac4
commit c44122ec8c
3 changed files with 24 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
projects/nat-archive-tool/src/nat/systems/MediaWikiSystem.kiss
View File

@@ -20,6 +20,8 @@
importHxFile "extern-files/python/import.hx" importHxFile "extern-files/python/import.hx"
langProjectFile "extern-files/python/requirements.txt") langProjectFile "extern-files/python/requirements.txt")
[:Map<String,Array<String>> params _ :String mediaWikiUrl _ :String maxLag (if maxLag (Std.string maxLag) "1") :Map<String,String> headers _] [:Map<String,Array<String>> params _ :String mediaWikiUrl _ :String maxLag (if maxLag (Std.string maxLag) "1") :Map<String,String> headers _]
(print "extern query call $params")
(let [response (let [response
(Requests.get mediaWikiUrl (Requests.get mediaWikiUrl
(let [innerParams (let [innerParams
@@ -30,8 +32,11 @@
]] ]]
(doFor =>param paramValues params (dictSet innerParams param (paramValues.join "|"))) (doFor =>param paramValues params (dictSet innerParams param (paramValues.join "|")))
innerParams) innerParams)
(object headers headers))] (object
headers headers
timeout 2))]
(assert response.ok) (assert response.ok)
(print "call finished: ${response.text}")
response.text)))) response.text))))
(method :Array<String> queryImageTitles [:Array<String> pageTitles] (method :Array<String> queryImageTitles [:Array<String> pageTitles]
@@ -43,7 +48,9 @@
(method :Array<String> queryImageUrls [:Array<String> imageTitles] (method :Array<String> queryImageUrls [:Array<String> imageTitles]
(flatten (flatten
(for =>_id image (the haxe.DynamicAccess<Dynamic> .pages .query (query [=>"titles" imageTitles =>"prop" ["imageinfo"] =>"iiprop" ["url"]])) (for =>_id image
(the haxe.DynamicAccess<Dynamic>
.pages .query (query [=>"titles" imageTitles =>"prop" ["imageinfo"] =>"iiprop" ["url"]]))
(if image.imageinfo (if image.imageinfo
(image.imageinfo.map ->image image.url) (image.imageinfo.map ->image image.url)
[])))) []))))

16
projects/nat-archive-tool/src/nat/systems/WikipediaImageSystem.kiss
View File

@@ -13,13 +13,15 @@
(readComponent archive e Name) (readComponent archive e Name)
:Array<String> wikipediaImageUrls :Array<String> wikipediaImageUrls
(queryImageUrls (queryImageTitles [title]))] (queryImageUrls (queryImageTitles [title]))]
(doFor url wikipediaImageUrls
(assertProcess "wget" ["--directory-prefix=${archive.archiveDir}" url]))
(addFiles archive e (addFiles archive e
(for url wikipediaImageUrls (for url wikipediaImageUrls
(joinPath archive.archiveDir (let [filePath
~(.replace (joinPath archive.archiveDir
(.urlDecode (url.withoutDirectory)) ~(.replace
// Some symbols shouldn't be decoded because they're invalid in file systems! (.urlDecode (url.withoutDirectory))
"\"" "%22")))) // Some symbols shouldn't be decoded because they're invalid in file systems!
"\"" "%22"))]
(print url "downloading")
(assertProcess "curl" ["--output" filePath url])
filePath)))
(addTags archive e ["wikipediaProcessed"]))) (addTags archive e ["wikipediaProcessed"])))

9
projects/requests-externs/src/requests_externs/Requests.hx
View File

@@ -5,11 +5,13 @@ import python.Dict;
import python.KwArgs; import python.KwArgs;
typedef NativeRequestKwArgs = { typedef NativeRequestKwArgs = {
?headers:Dict<String, String> ?headers:Dict<String, String>,
?timeout:Float
} }
typedef RequestKwArgs = { typedef RequestKwArgs = {
?headers:Map<String, String> ?headers:Map<String, String>,
?timeout:Float
} }
@:pythonImport("requests") @:pythonImport("requests")
@@ -36,7 +38,8 @@ class Requests {
if (kwArgs == null) if (kwArgs == null)
return null; return null;
return { return {
headers: mapToDict(kwArgs.headers) headers: mapToDict(kwArgs.headers),
timeout: kwArgs.timeout
}; };
} }
} }