Krow/Krow-workspace
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch '493-implement-rapid-order-creation-voice-text-in-client-mobile-app' into dev

Browse Source
This commit is contained in:
Achintha Isuru
2026-02-27 13:17:41 -05:00
parent 9d25fd44cc 2d4eee576c
commit e0d633d65d
23 changed files with 436 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
apps/mobile/apps/client/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java
View File

@@ -50,6 +50,11 @@ public final class GeneratedPluginRegistrant {
} catch (Exception e) {
Log.e(TAG, "Error registering plugin path_provider_android, io.flutter.plugins.pathprovider.PathProviderPlugin", e);
}
try {
flutterEngine.getPlugins().add(new com.llfbandit.record.RecordPlugin());
} catch (Exception e) {
Log.e(TAG, "Error registering plugin record_android, com.llfbandit.record.RecordPlugin", e);
}
try {
flutterEngine.getPlugins().add(new io.flutter.plugins.sharedpreferences.SharedPreferencesPlugin());
} catch (Exception e) {

4
apps/mobile/apps/client/linux/flutter/generated_plugin_registrant.cc
View File

@@ -7,12 +7,16 @@
#include "generated_plugin_registrant.h"
#include <file_selector_linux/file_selector_plugin.h>
#include <record_linux/record_linux_plugin.h>
#include <url_launcher_linux/url_launcher_plugin.h>
void fl_register_plugins(FlPluginRegistry* registry) {
g_autoptr(FlPluginRegistrar) file_selector_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin");
file_selector_plugin_register_with_registrar(file_selector_linux_registrar);
g_autoptr(FlPluginRegistrar) record_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "RecordLinuxPlugin");
record_linux_plugin_register_with_registrar(record_linux_registrar);
g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin");
url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar);

1
apps/mobile/apps/client/linux/flutter/generated_plugins.cmake
View File

@@ -4,6 +4,7 @@
list(APPEND FLUTTER_PLUGIN_LIST
file_selector_linux
record_linux
url_launcher_linux
)

3
apps/mobile/apps/client/windows/flutter/generated_plugin_registrant.cc
View File

@@ -9,6 +9,7 @@
#include <file_selector_windows/file_selector_windows.h>
#include <firebase_auth/firebase_auth_plugin_c_api.h>
#include <firebase_core/firebase_core_plugin_c_api.h>
#include <record_windows/record_windows_plugin_c_api.h>
#include <url_launcher_windows/url_launcher_windows.h>
void RegisterPlugins(flutter::PluginRegistry* registry) {
@@ -18,6 +19,8 @@ void RegisterPlugins(flutter::PluginRegistry* registry) {
registry->GetRegistrarForPlugin("FirebaseAuthPluginCApi"));
FirebaseCorePluginCApiRegisterWithRegistrar(
registry->GetRegistrarForPlugin("FirebaseCorePluginCApi"));
RecordWindowsPluginCApiRegisterWithRegistrar(
registry->GetRegistrarForPlugin("RecordWindowsPluginCApi"));
UrlLauncherWindowsRegisterWithRegistrar(
registry->GetRegistrarForPlugin("UrlLauncherWindows"));
}

1
apps/mobile/apps/client/windows/flutter/generated_plugins.cmake
View File

@@ -6,6 +6,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
file_selector_windows
firebase_auth
firebase_core
record_windows
url_launcher_windows
)

5
apps/mobile/apps/staff/android/app/src/main/java/io/flutter/plugins/GeneratedPluginRegistrant.java
View File

@@ -65,6 +65,11 @@ public final class GeneratedPluginRegistrant {
} catch (Exception e) {
Log.e(TAG, "Error registering plugin permission_handler_android, com.baseflow.permissionhandler.PermissionHandlerPlugin", e);
}
try {
flutterEngine.getPlugins().add(new com.llfbandit.record.RecordPlugin());
} catch (Exception e) {
Log.e(TAG, "Error registering plugin record_android, com.llfbandit.record.RecordPlugin", e);
}
try {
flutterEngine.getPlugins().add(new io.flutter.plugins.sharedpreferences.SharedPreferencesPlugin());
} catch (Exception e) {

4
apps/mobile/apps/staff/linux/flutter/generated_plugin_registrant.cc
View File

@@ -7,12 +7,16 @@
#include "generated_plugin_registrant.h"
#include <file_selector_linux/file_selector_plugin.h>
#include <record_linux/record_linux_plugin.h>
#include <url_launcher_linux/url_launcher_plugin.h>
void fl_register_plugins(FlPluginRegistry* registry) {
g_autoptr(FlPluginRegistrar) file_selector_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin");
file_selector_plugin_register_with_registrar(file_selector_linux_registrar);
g_autoptr(FlPluginRegistrar) record_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "RecordLinuxPlugin");
record_linux_plugin_register_with_registrar(record_linux_registrar);
g_autoptr(FlPluginRegistrar) url_launcher_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "UrlLauncherPlugin");
url_launcher_plugin_register_with_registrar(url_launcher_linux_registrar);

1
apps/mobile/apps/staff/linux/flutter/generated_plugins.cmake
View File

@@ -4,6 +4,7 @@
list(APPEND FLUTTER_PLUGIN_LIST
file_selector_linux
record_linux
url_launcher_linux
)

3
apps/mobile/apps/staff/windows/flutter/generated_plugin_registrant.cc
View File

@@ -11,6 +11,7 @@
#include <firebase_core/firebase_core_plugin_c_api.h>
#include <geolocator_windows/geolocator_windows.h>
#include <permission_handler_windows/permission_handler_windows_plugin.h>
#include <record_windows/record_windows_plugin_c_api.h>
#include <url_launcher_windows/url_launcher_windows.h>
void RegisterPlugins(flutter::PluginRegistry* registry) {
@@ -24,6 +25,8 @@ void RegisterPlugins(flutter::PluginRegistry* registry) {
registry->GetRegistrarForPlugin("GeolocatorWindows"));
PermissionHandlerWindowsPluginRegisterWithRegistrar(
registry->GetRegistrarForPlugin("PermissionHandlerWindowsPlugin"));
RecordWindowsPluginCApiRegisterWithRegistrar(
registry->GetRegistrarForPlugin("RecordWindowsPluginCApi"));
UrlLauncherWindowsRegisterWithRegistrar(
registry->GetRegistrarForPlugin("UrlLauncherWindows"));
}

1
apps/mobile/apps/staff/windows/flutter/generated_plugins.cmake
View File

@@ -8,6 +8,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
firebase_core
geolocator_windows
permission_handler_windows
record_windows
url_launcher_windows
)

3
apps/mobile/packages/core/lib/core.dart
View File

@@ -23,9 +23,12 @@ export 'src/services/api_service/core_api_services/llm/llm_service.dart';
export 'src/services/api_service/core_api_services/llm/llm_response.dart';
export 'src/services/api_service/core_api_services/verification/verification_service.dart';
export 'src/services/api_service/core_api_services/verification/verification_response.dart';
export 'src/services/api_service/core_api_services/rapid_order/rapid_order_service.dart';
export 'src/services/api_service/core_api_services/rapid_order/rapid_order_response.dart';
// Device Services
export 'src/services/device/camera/camera_service.dart';
export 'src/services/device/gallery/gallery_service.dart';
export 'src/services/device/file/file_picker_service.dart';
export 'src/services/device/file_upload/device_file_upload_service.dart';
export 'src/services/device/audio/audio_recorder_service.dart';

4
apps/mobile/packages/core/lib/src/core_module.dart
View File

@@ -29,6 +29,9 @@ class CoreModule extends Module {
() => VerificationService(i.get<BaseApiService>()),
);
i.addSingleton<LlmService>(() => LlmService(i.get<BaseApiService>()));
i.addSingleton<RapidOrderService>(
() => RapidOrderService(i.get<BaseApiService>()),
);
// 4. Register Device dependency
i.addSingleton<ImagePicker>(() => ImagePicker());
@@ -37,6 +40,7 @@ class CoreModule extends Module {
i.addSingleton<CameraService>(() => CameraService(i.get<ImagePicker>()));
i.addSingleton<GalleryService>(() => GalleryService(i.get<ImagePicker>()));
i.addSingleton<FilePickerService>(FilePickerService.new);
i.addSingleton<AudioRecorderService>(AudioRecorderService.new);
i.addSingleton<DeviceFileUploadService>(
() => DeviceFileUploadService(
cameraService: i.get<CameraService>(),

7
apps/mobile/packages/core/lib/src/services/api_service/core_api_services/core_api_endpoints.dart
View File

@@ -30,4 +30,11 @@ class CoreApiEndpoints {
/// Retry a verification job.
static String verificationRetry(String id) =>
'$baseUrl/core/verifications/$id/retry';
/// Transcribe audio to text for rapid orders.
static const String transcribeRapidOrder =
'$baseUrl/core/rapid-orders/transcribe';
/// Parse text to structured rapid order.
static const String parseRapidOrder = '$baseUrl/core/rapid-orders/parse';
}

239
apps/mobile/packages/core/lib/src/services/api_service/core_api_services/rapid_order/rapid_order_response.dart Normal file
View File

@@ -0,0 +1,239 @@
/// Response model for RAPID order transcription.
class RapidOrderTranscriptionResponse {
/// Creates a [RapidOrderTranscriptionResponse].
const RapidOrderTranscriptionResponse({
required this.transcript,
required this.confidence,
required this.language,
required this.warnings,
required this.model,
});
/// Factory to create [RapidOrderTranscriptionResponse] from JSON.
factory RapidOrderTranscriptionResponse.fromJson(Map<String, dynamic> json) {
return RapidOrderTranscriptionResponse(
transcript: json['transcript'] as String,
confidence: (json['confidence'] as num).toDouble(),
language: json['language'] as String,
warnings: List<String>.from(json['warnings'] as Iterable<dynamic>),
model: json['model'] as String,
);
}
/// The transcribed text.
final String transcript;
/// Confidence score (0.0 to 1.0).
final double confidence;
/// Language code.
final String language;
/// Any warnings from the transcription process.
final List<String> warnings;
/// The model name used for transcription.
final String model;
/// Converts the response to a JSON map.
Map<String, dynamic> toJson() {
return <String, dynamic>{
'transcript': transcript,
'confidence': confidence,
'language': language,
'warnings': warnings,
'model': model,
};
}
}
/// Response model for RAPID order parsing.
class RapidOrderParseResponse {
/// Creates a [RapidOrderParseResponse].
const RapidOrderParseResponse({
required this.parsed,
required this.missingFields,
required this.warnings,
required this.confidence,
required this.model,
});
/// Factory to create [RapidOrderParseResponse] from JSON.
factory RapidOrderParseResponse.fromJson(Map<String, dynamic> json) {
return RapidOrderParseResponse(
parsed: RapidOrderParsedData.fromJson(
json['parsed'] as Map<String, dynamic>,
),
missingFields: List<String>.from(
json['missingFields'] as Iterable<dynamic>,
),
warnings: List<String>.from(json['warnings'] as Iterable<dynamic>),
confidence: RapidOrderParseConfidence.fromJson(
json['confidence'] as Map<String, dynamic>,
),
model: json['model'] as String,
);
}
/// The parsed order data.
final RapidOrderParsedData parsed;
/// Fields that were identified as missing from the input.
final List<String> missingFields;
/// Any warnings from the parsing process.
final List<String> warnings;
/// Confidence scores.
final RapidOrderParseConfidence confidence;
/// The model name used for parsing.
final String model;
/// Converts the response to a JSON map.
Map<String, dynamic> toJson() {
return <String, dynamic>{
'parsed': parsed.toJson(),
'missingFields': missingFields,
'warnings': warnings,
'confidence': confidence.toJson(),
'model': model,
};
}
}
/// Parsed data for a rapid order.
class RapidOrderParsedData {
/// Creates a [RapidOrderParsedData].
const RapidOrderParsedData({
required this.orderType,
required this.isRapid,
required this.positions,
required this.sourceText,
this.startAt,
this.endAt,
this.durationMinutes,
this.locationHint,
this.notes,
});
/// Factory to create [RapidOrderParsedData] from JSON.
factory RapidOrderParsedData.fromJson(Map<String, dynamic> json) {
return RapidOrderParsedData(
orderType: json['orderType'] as String,
isRapid: json['isRapid'] as bool,
positions: (json['positions'] as List<dynamic>)
.map(
(dynamic e) =>
RapidOrderPosition.fromJson(e as Map<String, dynamic>),
)
.toList(),
sourceText: json['sourceText'] as String,
startAt: json['startAt'] as String?,
endAt: json['endAt'] as String?,
durationMinutes: json['durationMinutes'] as int?,
locationHint: json['locationHint'] as String?,
notes: json['notes'] as String?,
);
}
/// The type of order (typically ONE_TIME).
final String orderType;
/// Whether this is a rapid order.
final bool isRapid;
/// The requested positions.
final List<RapidOrderPosition> positions;
/// The source text that was parsed.
final String sourceText;
/// ISO datetime string for the start of the order.
final String? startAt;
/// ISO datetime string for the end of the order.
final String? endAt;
/// Duration in minutes.
final int? durationMinutes;
/// Hint about the location.
final String? locationHint;
/// Any notes captured from the input.
final String? notes;
/// Converts to a JSON map.
Map<String, dynamic> toJson() {
return <String, dynamic>{
'orderType': orderType,
'isRapid': isRapid,
'positions': positions.map((RapidOrderPosition e) => e.toJson()).toList(),
'sourceText': sourceText,
if (startAt != null) 'startAt': startAt,
if (endAt != null) 'endAt': endAt,
if (durationMinutes != null) 'durationMinutes': durationMinutes,
if (locationHint != null) 'locationHint': locationHint,
if (notes != null) 'notes': notes,
};
}
}
/// A position within a rapid order.
class RapidOrderPosition {
/// Creates a [RapidOrderPosition].
const RapidOrderPosition({required this.role, required this.count});
/// Factory to create [RapidOrderPosition] from JSON.
factory RapidOrderPosition.fromJson(Map<String, dynamic> json) {
return RapidOrderPosition(
role: json['role'] as String,
count: json['count'] as int,
);
}
/// The role name.
final String role;
/// Number of people needed for this role.
final int count;
/// Converts to a JSON map.
Map<String, dynamic> toJson() {
return <String, dynamic>{'role': role, 'count': count};
}
}
/// Confidence scores for a rapid order parse.
class RapidOrderParseConfidence {
/// Creates a [RapidOrderParseConfidence].
const RapidOrderParseConfidence({
required this.overall,
required this.fields,
});
/// Factory to create [RapidOrderParseConfidence] from JSON.
factory RapidOrderParseConfidence.fromJson(Map<String, dynamic> json) {
return RapidOrderParseConfidence(
overall: (json['overall'] as num).toDouble(),
fields: Map<String, double>.from(
(json['fields'] as Map<String, dynamic>).map(
(String k, dynamic v) =>
MapEntry<String, double>(k, (v as num).toDouble()),
),
),
);
}
/// Overall confidence score (0.0 to 1.0).
final double overall;
/// Confidence scores for specific fields.
final Map<String, double> fields;
/// Converts to a JSON map.
Map<String, dynamic> toJson() {
return <String, dynamic>{'overall': overall, 'fields': fields};
}
}

70
apps/mobile/packages/core/lib/src/services/api_service/core_api_services/rapid_order/rapid_order_service.dart Normal file
View File

@@ -0,0 +1,70 @@
import 'package:krow_domain/krow_domain.dart';
import '../core_api_endpoints.dart';
import 'rapid_order_response.dart';
/// Service for handling RAPID order operations (Transcription and Parsing).
class RapidOrderService extends BaseCoreService {
/// Creates a [RapidOrderService].
RapidOrderService(super.api);
/// Transcribes audio from a file to text for a RAPID order.
///
/// [audioFileUri] is the URI of the audio file to transcribe.
/// [locale] is the optional locale hint (default: 'en-US').
/// [promptHints] are optional domain hints to improve transcription quality.
Future<RapidOrderTranscriptionResponse> transcribeAudio({
required String audioFileUri,
String locale = 'en-US',
List<String>? promptHints,
}) async {
final ApiResponse res = await action(() async {
return api.post(
CoreApiEndpoints.transcribeRapidOrder,
data: <String, dynamic>{
'audioFileUri': audioFileUri,
'locale': locale,
if (promptHints != null) 'promptHints': promptHints,
},
);
});
if (res.code.startsWith('2')) {
return RapidOrderTranscriptionResponse.fromJson(
res.data as Map<String, dynamic>,
);
}
throw Exception(res.message);
}
/// Parses text into a structured RAPID order draft JSON.
///
/// [text] is the input text to parse.
/// [locale] is the optional locale hint (default: 'en-US').
/// [timezone] is the optional IANA timezone hint.
/// [now] is an optional current ISO datetime for reference.
Future<RapidOrderParseResponse> parseText({
required String text,
String locale = 'en-US',
String? timezone,
String? now,
}) async {
final ApiResponse res = await action(() async {
return api.post(
CoreApiEndpoints.parseRapidOrder,
data: <String, dynamic>{
'text': text,
'locale': locale,
if (timezone != null) 'timezone': timezone,
if (now != null) 'now': now,
},
);
});
if (res.code.startsWith('2')) {
return RapidOrderParseResponse.fromJson(res.data as Map<String, dynamic>);
}
throw Exception(res.message);
}
}

55
apps/mobile/packages/core/lib/src/services/device/audio/audio_recorder_service.dart Normal file
View File

@@ -0,0 +1,55 @@
import 'dart:io';
import 'package:path_provider/path_provider.dart';
import 'package:record/record.dart';
import 'package:krow_domain/krow_domain.dart';
/// Service for recording audio using the device microphone.
class AudioRecorderService extends BaseDeviceService {
/// Creates an [AudioRecorderService].
AudioRecorderService() : _recorder = AudioRecorder();
final AudioRecorder _recorder;
/// Starts recording audio to a temporary file.
///
/// Returns the path where the audio is being recorded.
Future<void> startRecording() async {
return action(() async {
if (await _recorder.hasPermission()) {
final Directory tempDir = await getTemporaryDirectory();
final String path =
'${tempDir.path}/rapid_order_audio_${DateTime.now().millisecondsSinceEpoch}.m4a';
// Configure the recording
const RecordConfig config = RecordConfig(
encoder: AudioEncoder.aacLc, // Good balance of quality and size
bitRate: 128000,
sampleRate: 44100,
);
await _recorder.start(config, path: path);
} else {
throw Exception('Microphone permission not granted');
}
});
}
/// Stops the current recording.
///
/// Returns the path to the recorded audio file, or null if no recording was active.
Future<String?> stopRecording() async {
return action(() async {
return await _recorder.stop();
});
}
/// Checks if the recorder is currently recording.
Future<bool> isRecording() async {
return await _recorder.isRecording();
}
/// Disposes the recorder resources.
void dispose() {
_recorder.dispose();
}
}

3
apps/mobile/packages/design_system/lib/src/ui_icons.dart
View File

@@ -285,4 +285,7 @@ class UiIcons {
/// Circle dollar icon
static const IconData circleDollar = _IconLib.circleDollarSign;
/// Microphone icon
static const IconData microphone = _IconLib.mic;
}

16
apps/mobile/packages/features/client/orders/create_order/lib/src/domain/usecases/transcribe_rapid_order_usecase.dart Normal file
View File

@@ -0,0 +1,16 @@
import '../repositories/client_create_order_repository_interface.dart';
/// Use case for transcribing audio for a rapid order.
class TranscribeRapidOrderUseCase {
/// Creates a [TranscribeRapidOrderUseCase].
TranscribeRapidOrderUseCase(this._repository);
final ClientCreateOrderRepositoryInterface _repository;
/// Executes the use case.
///
/// [audioPath] is the local path to the audio file.
Future<String> call(String audioPath) async {
return _repository.transcribeRapidOrder(audioPath);
}
}

4
apps/mobile/packages/features/client/orders/create_order/lib/src/presentation/widgets/rapid_order/rapid_order_view.dart
View File

@@ -29,7 +29,7 @@ class RapidOrderView extends StatelessWidget {
title: labels.success_title,
message: labels.success_message,
buttonLabel: labels.back_to_orders,
onDone: () => Modular.to.navigate(ClientPaths.orders),
onDone: () => Modular.to.toClientOrders(),
);
}
@@ -280,7 +280,7 @@ class _RapidOrderActions extends StatelessWidget {
Expanded(
child: UiButton.secondary(
text: isListening ? labels.listening : labels.speak,
leadingIcon: UiIcons.bell, // Placeholder for mic
leadingIcon: UiIcons.microphone,
onPressed: () => BlocProvider.of<RapidOrderBloc>(
context,
).add(const RapidOrderVoiceToggled()),

4
apps/mobile/packages/features/client/orders/orders_common/linux/flutter/generated_plugin_registrant.cc
View File

@@ -7,9 +7,13 @@
#include "generated_plugin_registrant.h"
#include <file_selector_linux/file_selector_plugin.h>
#include <record_linux/record_linux_plugin.h>
void fl_register_plugins(FlPluginRegistry* registry) {
g_autoptr(FlPluginRegistrar) file_selector_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin");
file_selector_plugin_register_with_registrar(file_selector_linux_registrar);
g_autoptr(FlPluginRegistrar) record_linux_registrar =
fl_plugin_registry_get_registrar_for_plugin(registry, "RecordLinuxPlugin");
record_linux_plugin_register_with_registrar(record_linux_registrar);
}

1
apps/mobile/packages/features/client/orders/orders_common/linux/flutter/generated_plugins.cmake
View File

@@ -4,6 +4,7 @@
list(APPEND FLUTTER_PLUGIN_LIST
file_selector_linux
record_linux
)
list(APPEND FLUTTER_FFI_PLUGIN_LIST

3
apps/mobile/packages/features/client/orders/orders_common/windows/flutter/generated_plugin_registrant.cc
View File

@@ -9,6 +9,7 @@
#include <file_selector_windows/file_selector_windows.h>
#include <firebase_auth/firebase_auth_plugin_c_api.h>
#include <firebase_core/firebase_core_plugin_c_api.h>
#include <record_windows/record_windows_plugin_c_api.h>
void RegisterPlugins(flutter::PluginRegistry* registry) {
FileSelectorWindowsRegisterWithRegistrar(
@@ -17,4 +18,6 @@ void RegisterPlugins(flutter::PluginRegistry* registry) {
registry->GetRegistrarForPlugin("FirebaseAuthPluginCApi"));
FirebaseCorePluginCApiRegisterWithRegistrar(
registry->GetRegistrarForPlugin("FirebaseCorePluginCApi"));
RecordWindowsPluginCApiRegisterWithRegistrar(
registry->GetRegistrarForPlugin("RecordWindowsPluginCApi"));
}

1
apps/mobile/packages/features/client/orders/orders_common/windows/flutter/generated_plugins.cmake
View File

@@ -6,6 +6,7 @@ list(APPEND FLUTTER_PLUGIN_LIST
file_selector_windows
firebase_auth
firebase_core
record_windows
)
list(APPEND FLUTTER_FFI_PLUGIN_LIST