rough screenreader support in SimpleWindow (only interactive elements)

2024-03-11 13:07:29 +01:00
parent b6e9c43dd5
commit ab4b2f46fa
5 changed files with 76 additions and 14 deletions
--- a/simplewindow-test/haxe_libraries/kiss-tools.hxml
+++ b/simplewindow-test/haxe_libraries/kiss-tools.hxml
@@ -1,4 +1,4 @@
-# @install: lix --silent download "gh://github.com/kiss-lang/kiss-tools#1a9199cf72dcbc7214a43709c7102766a68206f6" into kiss-tools/0.0.0/github/1a9199cf72dcbc7214a43709c7102766a68206f6
+# @install: lix --silent download "gh://github.com/kiss-lang/kiss-tools#eacc6bd02bccb3b997fd21825f29a736b7dfb159" into kiss-tools/0.0.0/github/eacc6bd02bccb3b997fd21825f29a736b7dfb159
 -lib kiss
-cp ${HAXE_LIBCACHE}/kiss-tools/0.0.0/github/1a9199cf72dcbc7214a43709c7102766a68206f6/src/
+-cp ${HAXE_LIBCACHE}/kiss-tools/0.0.0/github/eacc6bd02bccb3b997fd21825f29a736b7dfb159/src/
 -D kiss-tools=0.0.0
--- a/simplewindow-test/haxe_libraries/kiss.hxml
+++ b/simplewindow-test/haxe_libraries/kiss.hxml
@@ -1,12 +1,12 @@
-# @install: lix --silent download "gh://github.com/kiss-lang/kiss#4a95750fedbfca54f91d8bcf99cb8c9fd38cf6b9" into kiss/0.0.1/github/4a95750fedbfca54f91d8bcf99cb8c9fd38cf6b9
-# @run: haxelib run-dir kiss "${HAXE_LIBCACHE}/kiss/0.0.1/github/4a95750fedbfca54f91d8bcf99cb8c9fd38cf6b9"
+# @install: lix --silent download "gh://github.com/kiss-lang/kiss#de4f841d4d57e8b6239418e90f104aa0209b1880" into kiss/0.0.1/github/de4f841d4d57e8b6239418e90f104aa0209b1880
+# @run: haxelib run-dir kiss "${HAXE_LIBCACHE}/kiss/0.0.1/github/de4f841d4d57e8b6239418e90f104aa0209b1880"
 -lib haxe-strings
 -lib hscript
 -lib tink_json
 -lib tink_macro
 -lib tink_syntaxhub
 -lib uuid
-cp ${HAXE_LIBCACHE}/kiss/0.0.1/github/4a95750fedbfca54f91d8bcf99cb8c9fd38cf6b9/src
+-cp ${HAXE_LIBCACHE}/kiss/0.0.1/github/de4f841d4d57e8b6239418e90f104aa0209b1880/src
 -D kiss=0.0.1
 -w -WUnusedPattern
 --macro kiss.KissFrontend.use()
--- a/simplewindow-test/source/PlayState.kiss
+++ b/simplewindow-test/source/PlayState.kiss
@@ -15,7 +15,7 @@
    (showMenu))

 (method :Void showMenu []
-    (SimpleWindow.promptForChoice "Testing SimpleWindow" ["Vertical" "Menu" "Feedback" "Quit"]
+    (SimpleWindow.promptForChoiceV2 "Testing SimpleWindow" ["Vertical" "Menu" "Feedback" "Quit"]
        ->:Void choice 
            (case choice
                ("Vertical"
@@ -26,7 +26,8 @@
                    (FeedbackWindow.collectFeedback showMenu))
                ("Quit"
                    (Sys.exit 0))
-                (never otherwise))))
+                (never otherwise))
+        (object screenReaderAudioFolder "assets/screen-reader-audio")))

 (method subMenu []
    (let [window (new SimpleWindow "SubMenu")]
--- a/src/kiss_flixel/SimpleWindow.hx
+++ b/src/kiss_flixel/SimpleWindow.hx
@@ -9,6 +9,7 @@ import kiss_flixel.KissInputText;
 import flixel.text.FlxText;
 import flixel.FlxSprite;
 import flixel.util.FlxColor;
+import flixel.sound.FlxSound;
 import flixel.math.FlxRect;
 import flixel.group.FlxGroup;
 import kiss_flixel.FlxKeyShortcutHandler;
@@ -20,6 +21,9 @@ import flixel.input.mouse.FlxMouseEvent;
 import kiss_flixel.KissExtendedSprite;
 import flixel.addons.plugin.FlxMouseControl;

+using haxe.io.Path;
+using StringTools;
+
 typedef ShortcutAction = Void->Void;
 typedef Action = FlxSprite->Void;

--- a/src/kiss_flixel/SimpleWindow.kiss
+++ b/src/kiss_flixel/SimpleWindow.kiss
@@ -5,6 +5,9 @@
 (var :kiss.List<SimpleWindow> windowStack [])
 (var &mut :flixel.FlxCamera defaultCamera null)

+(savedVar :Float screenReaderVolume 1.0)
+(savedVar :Bool screenReaderEnabled true)
+
 (prop :FlxCamera controlCamera)

 (prop &mut keyboardEnabled true)
@@ -41,7 +44,45 @@
                (set selectionMarker.y controlToSelect.y)
                (set selectionMarker.x (- controlToSelect.x selectionMarker.width textSize)))

-            // TODO play screenreader of the text
+            // TODO method to play screenreader of all non-interactive text with a cc of playing the first interactive text audio
+
+            // TODO method to prompt explaining the screenreader and optionally asking to enable/disable/configure volume?
+
+            // play screenreader of the text (unless hidden?)
+            (method :Void playScreenReaderTextFor [control &opt :Void->Void cc]
+                (typeCase [control]
+                    ([:FlxText text]
+                        (playScreenReaderText text.text cc))
+                    (otherwise (print "Can't determine text for ScreenReader of SimpleWindow control $control"))))
+
+            (method :Void playScreenReaderText [text &opt :Void->Void cc]
+                // stop any currently playing text
+                // TODO is it right that the cc will never be called?
+                (currentAudio?.stop)
+                
+                (if (and screenReaderEnabled (< 0 screenReaderVolume))
+                        {
+                            (ifLet [audio (dictGet screenReaderAudio text)]
+                                    (playAudio audio cc)
+                                {
+                                    (doFor =>key audio screenReaderAudio
+                                        (when (text.startsWith key)
+                                            (playAudio audio cc)
+                                            (return)))
+                                    (print "No audio found for SimpleWindow text `${text}`")
+                                    (when cc (cc))
+                                })
+                        }
+                    (when cc (cc))))
+
+            (prop &mut :FlxSound currentAudio)
+            (method playAudio [:FlxSound sound &opt :Void->Void cc]
+                (set sound.volume screenReaderVolume)
+                (set sound.onComplete cc)
+                (set currentAudio sound)
+                (sound.play))
+
+            (playScreenReaderTextFor controlToSelect)

            (whenLet [onSelect (dictGet _onSelectEvents controlToSelect)]
                (onSelect controlToSelect))
@@ -110,7 +151,8 @@
            :String _upKey :String _downKey
            :String _enterKey
            :ShortcutAction _onClose
-            :FlxSprite _selectionMarker]
+            :FlxSprite _selectionMarker
+            :String _screenReaderAudioFolder]
    
    [:String title (or _title "")
        &mut :Float nextControlX 0
@@ -132,10 +174,20 @@
        // UI key controls are still available. it also handles left and right.
        :FlxKeyShortcutHandler<ShortcutAction> xHandler (new FlxKeyShortcutHandler)
        :Int _width (Std.int (* FlxG.width (or percentWidth 0.5)))
-        :Int _height (Std.int (* FlxG.height (or percentHeight 0.5)))]
+        :Int _height (Std.int (* FlxG.height (or percentHeight 0.5)))
+        :String screenReaderAudioFolder _screenReaderAudioFolder
+        :Map<String,FlxSound> screenReaderAudio (new Map)]
    
    (assert FlxG.camera "SimpleWindow.new() must be called in or after create()")
    (super 0 0)
+    
+    // TODO this relies on readDirectory() which can't be done in JavaScript for HTML5 games
+    (#when sys
+        (when screenReaderAudioFolder
+            (let [audioFiles (sys.FileSystem.readDirectory screenReaderAudioFolder)]
+                (doFor file audioFiles
+                    (dictSet screenReaderAudio (file.withoutExtension) (FlxG.sound.load (joinPath screenReaderAudioFolder file)))))))
+
    (when defaultCamera (set this.cameras [defaultCamera]))
    (makeGraphic
        _width
@@ -649,7 +701,8 @@
                                    :Float percentHeight
                                    :Bool xButton
                                    :String xKey
-                                    :String enterKey]
+                                    :String enterKey
+                                    :String screenReaderAudioFolder]
    (promptForChoice
        message
        ["OK"] 
@@ -668,7 +721,8 @@
        enterKey
        onDismiss
        false
-        true))
+        true
+        screenReaderAudioFolder))

 (function :SimpleWindow promptForChoice <>[T] [:String prompt
                                            :Array<T> choices
@@ -687,8 +741,9 @@
                                            :String enterKey
                                            :ShortcutAction onClose
                                            :Bool noShortcuts
-                                            :Bool wrapPrompt]
-(let [window (new SimpleWindow (unless wrapPrompt prompt) bgColor titleColor percentWidth percentHeight xButton xKey leftKey rightKey upKey downKey enterKey onClose)
+                                            :Bool wrapPrompt
+                                            :String screenReaderAudioFolder]
+(let [window (new SimpleWindow (unless wrapPrompt prompt) bgColor titleColor percentWidth percentHeight xButton xKey leftKey rightKey upKey downKey enterKey onClose defaultSelectionMarker screenReaderAudioFolder)
        choiceColor (or choiceColor titleColor FlxColor.WHITE)]
    (when wrapPrompt
        (window.makeWrappedText prompt titleColor true))
@@ -703,6 +758,8 @@
    (window.show)
    window))

+(redefineWithObjectArgs promptForChoice promptForChoiceV2 [prompt choices onChoice])
+
 (function :SimpleWindow promptForString [:String prompt
                                        :String->Void onChoice
                                        &opt :FlxColor bgColor