From 3b50708dda552e8474ce249c3768d01a2b084315 Mon Sep 17 00:00:00 2001 From: abhimanyupandian Date: Sun, 23 Nov 2025 22:39:16 +0530 Subject: [PATCH] feat: example with flutter sdk --- flutter/.gitignore | 45 ++ flutter/.metadata | 30 ++ flutter/README.md | 23 + flutter/analysis_options.yaml | 28 + flutter/assets | 1 + flutter/lib/helper.dart | 486 ++++++++++++++++++ flutter/lib/main.dart | 363 +++++++++++++ flutter/macos/.gitignore | 7 + flutter/macos/Flutter/Flutter-Debug.xcconfig | 2 + .../macos/Flutter/Flutter-Release.xcconfig | 2 + .../Flutter/GeneratedPluginRegistrant.swift | 16 + flutter/macos/Podfile | 45 ++ flutter/macos/Podfile.lock | 54 ++ flutter/macos/Runner/AppDelegate.swift | 13 + .../AppIcon.appiconset/Contents.json | 68 +++ .../AppIcon.appiconset/app_icon_1024.png | Bin 0 -> 102994 bytes .../AppIcon.appiconset/app_icon_128.png | Bin 0 -> 5680 bytes .../AppIcon.appiconset/app_icon_16.png | Bin 0 -> 520 bytes .../AppIcon.appiconset/app_icon_256.png | Bin 0 -> 14142 bytes .../AppIcon.appiconset/app_icon_32.png | Bin 0 -> 1066 bytes .../AppIcon.appiconset/app_icon_512.png | Bin 0 -> 36406 bytes .../AppIcon.appiconset/app_icon_64.png | Bin 0 -> 2218 bytes flutter/macos/Runner/Base.lproj/MainMenu.xib | 343 ++++++++++++ flutter/macos/Runner/Configs/AppInfo.xcconfig | 14 + flutter/macos/Runner/Configs/Debug.xcconfig | 2 + flutter/macos/Runner/Configs/Release.xcconfig | 2 + .../macos/Runner/Configs/Warnings.xcconfig | 13 + .../macos/Runner/DebugProfile.entitlements | 12 + flutter/macos/Runner/Info.plist | 32 ++ flutter/macos/Runner/MainFlutterWindow.swift | 15 + flutter/macos/Runner/Release.entitlements | 8 + flutter/macos/RunnerTests/RunnerTests.swift | 12 + flutter/pubspec.lock | 418 +++++++++++++++ flutter/pubspec.yaml | 26 + 34 files changed, 2080 insertions(+) create mode 100644 flutter/.gitignore create mode 100644 flutter/.metadata create mode 100644 flutter/README.md create mode 100644 flutter/analysis_options.yaml create mode 120000 flutter/assets create mode 100644 flutter/lib/helper.dart create mode 100644 flutter/lib/main.dart create mode 100644 flutter/macos/.gitignore create mode 100644 flutter/macos/Flutter/Flutter-Debug.xcconfig create mode 100644 flutter/macos/Flutter/Flutter-Release.xcconfig create mode 100644 flutter/macos/Flutter/GeneratedPluginRegistrant.swift create mode 100644 flutter/macos/Podfile create mode 100644 flutter/macos/Podfile.lock create mode 100644 flutter/macos/Runner/AppDelegate.swift create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/Contents.json create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_1024.png create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_128.png create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_16.png create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_256.png create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_32.png create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_512.png create mode 100644 flutter/macos/Runner/Assets.xcassets/AppIcon.appiconset/app_icon_64.png create mode 100644 flutter/macos/Runner/Base.lproj/MainMenu.xib create mode 100644 flutter/macos/Runner/Configs/AppInfo.xcconfig create mode 100644 flutter/macos/Runner/Configs/Debug.xcconfig create mode 100644 flutter/macos/Runner/Configs/Release.xcconfig create mode 100644 flutter/macos/Runner/Configs/Warnings.xcconfig create mode 100644 flutter/macos/Runner/DebugProfile.entitlements create mode 100644 flutter/macos/Runner/Info.plist create mode 100644 flutter/macos/Runner/MainFlutterWindow.swift create mode 100644 flutter/macos/Runner/Release.entitlements create mode 100644 flutter/macos/RunnerTests/RunnerTests.swift create mode 100644 flutter/pubspec.lock create mode 100644 flutter/pubspec.yaml diff --git a/flutter/.gitignore b/flutter/.gitignore new file mode 100644 index 0000000..3820a95 --- /dev/null +++ b/flutter/.gitignore @@ -0,0 +1,45 @@ +# Miscellaneous +*.class +*.log +*.pyc +*.swp +.DS_Store +.atom/ +.build/ +.buildlog/ +.history +.svn/ +.swiftpm/ +migrate_working_dir/ + +# IntelliJ related +*.iml +*.ipr +*.iws +.idea/ + +# The .vscode folder contains launch configuration and tasks you configure in +# VS Code which you may wish to be included in version control, so this line +# is commented out by default. +#.vscode/ + +# Flutter/Dart/Pub related +**/doc/api/ +**/ios/Flutter/.last_build_id +.dart_tool/ +.flutter-plugins-dependencies +.pub-cache/ +.pub/ +/build/ +/coverage/ + +# Symbolication related +app.*.symbols + +# Obfuscation related +app.*.map.json + +# Android Studio will place build artifacts here +/android/app/debug +/android/app/profile +/android/app/release diff --git a/flutter/.metadata b/flutter/.metadata new file mode 100644 index 0000000..79bbc85 --- /dev/null +++ b/flutter/.metadata @@ -0,0 +1,30 @@ +# This file tracks properties of this Flutter project. +# Used by Flutter tool to assess capabilities and perform upgrades etc. +# +# This file should be version controlled and should not be manually edited. + +version: + revision: "19074d12f7eaf6a8180cd4036a430c1d76de904e" + channel: "stable" + +project_type: app + +# Tracks metadata for the flutter migrate command +migration: + platforms: + - platform: root + create_revision: 19074d12f7eaf6a8180cd4036a430c1d76de904e + base_revision: 19074d12f7eaf6a8180cd4036a430c1d76de904e + - platform: macos + create_revision: 19074d12f7eaf6a8180cd4036a430c1d76de904e + base_revision: 19074d12f7eaf6a8180cd4036a430c1d76de904e + + # User provided section + + # List of Local paths (relative to this file) that should be + # ignored by the migrate tool. + # + # Files that are not part of the templates will be ignored by default. + unmanaged_files: + - 'lib/main.dart' + - 'ios/Runner.xcodeproj/project.pbxproj' diff --git a/flutter/README.md b/flutter/README.md new file mode 100644 index 0000000..9225fea --- /dev/null +++ b/flutter/README.md @@ -0,0 +1,23 @@ +# Supertonic Flutter Example + +This example demonstrates how to use Supertonic in a Flutter application using ONNX Runtime. + +> **Note:** This project uses the `flutter_onnxruntime` package ([https://pub.dev/packages/flutter_onnxruntime](https://pub.dev/packages/flutter_onnxruntime)). At the moment, only the macOS platform has been tested. Although the flutter_onnxruntime package supports several other platforms, they have not been tested in this project yet and may require additional verification. + + +## šŸ“° Update News + +**2025.11.23** - Added and tested macos support. + +## Requirements + +- Flutter SDK version ^3.5.0 + +## Running the Demo + +```bash +flutter clean +flutter pub get +flutter run -d macos +``` + diff --git a/flutter/analysis_options.yaml b/flutter/analysis_options.yaml new file mode 100644 index 0000000..0d29021 --- /dev/null +++ b/flutter/analysis_options.yaml @@ -0,0 +1,28 @@ +# This file configures the analyzer, which statically analyzes Dart code to +# check for errors, warnings, and lints. +# +# The issues identified by the analyzer are surfaced in the UI of Dart-enabled +# IDEs (https://dart.dev/tools#ides-and-editors). The analyzer can also be +# invoked from the command line by running `flutter analyze`. + +# The following line activates a set of recommended lints for Flutter apps, +# packages, and plugins designed to encourage good coding practices. +include: package:flutter_lints/flutter.yaml + +linter: + # The lint rules applied to this project can be customized in the + # section below to disable rules from the `package:flutter_lints/flutter.yaml` + # included above or to enable additional rules. A list of all available lints + # and their documentation is published at https://dart.dev/lints. + # + # Instead of disabling a lint rule for the entire project in the + # section below, it can also be suppressed for a single line of code + # or a specific dart file by using the `// ignore: name_of_lint` and + # `// ignore_for_file: name_of_lint` syntax on the line or in the file + # producing the lint. + rules: + # avoid_print: false # Uncomment to disable the `avoid_print` rule + # prefer_single_quotes: true # Uncomment to enable the `prefer_single_quotes` rule + +# Additional information about this file can be found at +# https://dart.dev/guides/language/analysis-options diff --git a/flutter/assets b/flutter/assets new file mode 120000 index 0000000..ec2e4be --- /dev/null +++ b/flutter/assets @@ -0,0 +1 @@ +../assets \ No newline at end of file diff --git a/flutter/lib/helper.dart b/flutter/lib/helper.dart new file mode 100644 index 0000000..2d66680 --- /dev/null +++ b/flutter/lib/helper.dart @@ -0,0 +1,486 @@ +import 'dart:io'; +import 'dart:convert'; +import 'dart:math' as math; +import 'dart:typed_data'; +import 'package:flutter/services.dart' show rootBundle; +import 'package:flutter_onnxruntime/flutter_onnxruntime.dart'; +import 'package:logger/logger.dart'; +import 'package:path_provider/path_provider.dart'; + +final logger = Logger( + printer: PrettyPrinter(methodCount: 0, errorMethodCount: 5, lineLength: 80), +); + +class UnicodeProcessor { + final Map indexer; + + UnicodeProcessor._(this.indexer); + + static Future load(String path) async { + final json = jsonDecode( + path.startsWith('assets/') + ? await rootBundle.loadString(path) + : File(path).readAsStringSync(), + ); + + final indexer = json is List + ? { + for (var i = 0; i < json.length; i++) + if (json[i] is int && json[i] >= 0) i: json[i] as int + } + : (json as Map) + .map((k, v) => MapEntry(int.parse(k), v as int)); + + return UnicodeProcessor._(indexer); + } + + Map call(List textList) { + final lengths = textList.map((t) => t.length).toList(); + final maxLen = lengths.reduce(math.max); + + final textIds = textList.map((text) { + final row = List.filled(maxLen, 0); + final runes = text.runes.toList(); + for (var i = 0; i < runes.length; i++) { + row[i] = indexer[runes[i]] ?? 0; + } + return row; + }).toList(); + + return {'textIds': textIds, 'textMask': _lengthToMask(lengths)}; + } + + List>> _lengthToMask(List lengths, [int? maxLen]) { + maxLen ??= lengths.reduce(math.max); + return lengths + .map((len) => [List.generate(maxLen!, (i) => i < len ? 1.0 : 0.0)]) + .toList(); + } +} + +class Style { + final OrtValue ttl, dp; + final List ttlShape, dpShape; + Style(this.ttl, this.dp, this.ttlShape, this.dpShape); +} + +class TextToSpeech { + final Map cfgs; + final UnicodeProcessor textProcessor; + final OrtSession dpOrt, textEncOrt, vectorEstOrt, vocoderOrt; + final int sampleRate, baseChunkSize, chunkCompressFactor, ldim; + + TextToSpeech(this.cfgs, this.textProcessor, this.dpOrt, this.textEncOrt, + this.vectorEstOrt, this.vocoderOrt) + : sampleRate = cfgs['ae']['sample_rate'], + baseChunkSize = cfgs['ae']['base_chunk_size'], + chunkCompressFactor = cfgs['ttl']['chunk_compress_factor'], + ldim = cfgs['ttl']['latent_dim']; + + Future> call(String text, Style style, int totalStep, + {double speed = 1.05, double silenceDuration = 0.3}) async { + final chunks = _chunkText(text); + List? wavCat; + double durCat = 0; + + for (final chunk in chunks) { + final result = await _infer([chunk], style, totalStep, speed: speed); + final wav = _safeCast(result['wav']); + final duration = _safeCast(result['duration']); + + if (wavCat == null) { + wavCat = wav; + durCat = duration[0]; + } else { + wavCat = [ + ...wavCat, + ...List.filled((silenceDuration * sampleRate).floor(), 0.0), + ...wav + ]; + durCat += duration[0] + silenceDuration; + } + } + + return { + 'wav': wavCat, + 'duration': [durCat] + }; + } + + Future> _infer( + List textList, Style style, int totalStep, + {double speed = 1.05}) async { + final bsz = textList.length; + final result = textProcessor.call(textList); + + final textIdsRaw = result['textIds']; + final textIds = textIdsRaw is List> + ? textIdsRaw + : (textIdsRaw as List).map((row) => (row as List).cast()).toList(); + + final textMaskRaw = result['textMask']; + final textMask = textMaskRaw is List>> + ? textMaskRaw + : (textMaskRaw as List) + .map((batch) => (batch as List) + .map((row) => (row as List).cast()) + .toList()) + .toList(); + + final textIdsShape = [bsz, textIds[0].length]; + final textMaskShape = [bsz, 1, textMask[0][0].length]; + final textMaskTensor = await _toTensor(textMask, textMaskShape); + + final dpResult = await dpOrt.run({ + 'text_ids': await _intToTensor(textIds, textIdsShape), + 'style_dp': style.dp, + 'text_mask': textMaskTensor, + }); + final durOnnx = _safeCast(await dpResult.values.first.asList()); + final scaledDur = durOnnx.map((d) => d / speed).toList(); + + final textEncResult = await textEncOrt.run({ + 'text_ids': await _intToTensor(textIds, textIdsShape), + 'style_ttl': style.ttl, + 'text_mask': textMaskTensor, + }); + + final latentData = _sampleNoisyLatent(scaledDur); + final noisyLatentRaw = latentData['noisyLatent']; + var noisyLatent = noisyLatentRaw is List>> + ? noisyLatentRaw + : (noisyLatentRaw as List) + .map((batch) => (batch as List) + .map((row) => (row as List).cast()) + .toList()) + .toList(); + + final latentMaskRaw = latentData['latentMask']; + final latentMask = latentMaskRaw is List>> + ? latentMaskRaw + : (latentMaskRaw as List) + .map((batch) => (batch as List) + .map((row) => (row as List).cast()) + .toList()) + .toList(); + + final latentShape = [bsz, noisyLatent[0].length, noisyLatent[0][0].length]; + final latentMaskTensor = + await _toTensor(latentMask, [bsz, 1, latentMask[0][0].length]); + + final totalStepTensor = + await _scalarToTensor(List.filled(bsz, totalStep.toDouble()), [bsz]); + + // Denoising loop + for (var step = 0; step < totalStep; step++) { + final result = await vectorEstOrt.run({ + 'noisy_latent': await _toTensor(noisyLatent, latentShape), + 'text_emb': textEncResult.values.first, + 'style_ttl': style.ttl, + 'text_mask': textMaskTensor, + 'latent_mask': latentMaskTensor, + 'total_step': totalStepTensor, + 'current_step': + await _scalarToTensor(List.filled(bsz, step.toDouble()), [bsz]), + }); + + final denoisedRaw = await result.values.first.asList(); + final denoised = denoisedRaw is List + ? denoisedRaw + : _safeCast(denoisedRaw); + var idx = 0; + for (var b = 0; b < noisyLatent.length; b++) { + for (var d = 0; d < noisyLatent[b].length; d++) { + for (var t = 0; t < noisyLatent[b][d].length; t++) { + noisyLatent[b][d][t] = denoised[idx++]; + } + } + } + } + + final vocoderResult = await vocoderOrt + .run({'latent': await _toTensor(noisyLatent, latentShape)}); + final wavRaw = await vocoderResult.values.first.asList(); + final wav = wavRaw is List ? wavRaw : _safeCast(wavRaw); + + return {'wav': wav, 'duration': scaledDur}; + } + + Map _sampleNoisyLatent(List duration) { + final wavLenMax = duration.reduce(math.max) * sampleRate; + final wavLengths = duration.map((d) => (d * sampleRate).floor()).toList(); + final chunkSize = baseChunkSize * chunkCompressFactor; + final latentLen = ((wavLenMax + chunkSize - 1) / chunkSize).floor(); + final latentDim = ldim * chunkCompressFactor; + + final random = math.Random(); + final noisyLatent = List.generate( + duration.length, + (_) => List.generate( + latentDim, + (_) => List.generate(latentLen, (_) { + final u1 = math.max(1e-10, random.nextDouble()); + final u2 = random.nextDouble(); + return math.sqrt(-2.0 * math.log(u1)) * math.cos(2.0 * math.pi * u2); + }), + ), + ); + + final latentMask = _getLatentMask(wavLengths); + + for (var b = 0; b < noisyLatent.length; b++) { + for (var d = 0; d < noisyLatent[b].length; d++) { + for (var t = 0; t < noisyLatent[b][d].length; t++) { + noisyLatent[b][d][t] *= latentMask[b][0][t]; + } + } + } + + return {'noisyLatent': noisyLatent, 'latentMask': latentMask}; + } + + List>> _getLatentMask(List wavLengths) { + final latentSize = baseChunkSize * chunkCompressFactor; + final latentLengths = wavLengths + .map((len) => ((len + latentSize - 1) / latentSize).floor()) + .toList(); + final maxLen = latentLengths.reduce(math.max); + return latentLengths + .map((len) => [List.generate(maxLen, (i) => i < len ? 1.0 : 0.0)]) + .toList(); + } + + List _chunkText(String text, {int maxLen = 300}) { + final paragraphs = text + .trim() + .split(RegExp(r'\n\s*\n+')) + .where((p) => p.trim().isNotEmpty) + .toList(); + + final chunks = []; + for (var paragraph in paragraphs) { + paragraph = paragraph.trim(); + if (paragraph.isEmpty) continue; + + final sentences = paragraph.split(RegExp( + r'(? _safeCast(dynamic raw) { + if (raw is List) return raw; + if (raw is List) { + if (raw.isNotEmpty && raw.first is List) { + return _flattenList(raw); + } + if (T == double) { + return raw + .map((e) => e is num ? e.toDouble() : double.parse(e.toString())) + .toList() as List; + } + return raw.cast(); + } + throw Exception('Cannot convert $raw to List<$T>'); + } + + List _flattenList(dynamic list) { + if (list is List) { + return list.expand((e) => _flattenList(e)).toList(); + } + if (T == double && list is num) { + return [list.toDouble()] as List; + } + return [list as T]; + } + + Future _toTensor(dynamic array, List dims) async { + final flat = _flattenList(array); + return await OrtValue.fromList(Float32List.fromList(flat), dims); + } + + Future _scalarToTensor(List array, List dims) async { + return await OrtValue.fromList(Float32List.fromList(array), dims); + } + + Future _intToTensor(List> array, List dims) async { + final flat = array.expand((row) => row).toList(); + return await OrtValue.fromList(Int64List.fromList(flat), dims); + } +} + +Future loadTextToSpeech(String onnxDir, + {bool useGpu = false}) async { + if (useGpu) throw Exception('GPU mode not supported yet'); + + logger.i('Loading TTS models from $onnxDir'); + + final cfgs = await _loadCfgs(onnxDir); + final sessions = await _loadOnnxAll(onnxDir); + final textProcessor = + await UnicodeProcessor.load('$onnxDir/unicode_indexer.json'); + + logger.i('TTS models loaded successfully'); + + return TextToSpeech( + cfgs, + textProcessor, + sessions['dpOrt']!, + sessions['textEncOrt']!, + sessions['vectorEstOrt']!, + sessions['vocoderOrt']!, + ); +} + +Future