diff --git a/packages/devtools_app/lib/src/screens/network/har_builder.dart b/packages/devtools_app/lib/src/screens/network/har_builder.dart new file mode 100644 index 00000000000..b728b590b6b --- /dev/null +++ b/packages/devtools_app/lib/src/screens/network/har_builder.dart @@ -0,0 +1,174 @@ +// Copyright 2019 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import '../../shared/analytics/constants.dart'; +import '../../shared/http/http_request_data.dart'; +import '../../shared/utils.dart'; + +/// Builds a HAR (HTTP Archive) object from a list of HTTP requests. +/// +/// The HAR format is a JSON-based format used for logging a web browser's +/// interaction with a site. It is useful for performance analysis and +/// debugging. This function constructs the HAR object based on the 1.2 +/// specification. +/// +/// For more details on the HAR format, see the [HAR 1.2 Specification](https://github.com/ahmadnassri/har-spec/blob/master/versions/1.2.md). +/// +/// Parameters: +/// - [httpRequests]: A list of DartIOHttpRequestData data. +/// +/// Returns: +/// - A Map representing the HAR object. +Map buildHar(List httpRequests) { + // Build the creator + final creator = { + NetworkEventKeys.name: NetworkEventDefaults.creatorName, + NetworkEventKeys.creatorVersion: devToolsVersion, + }; + + // Build the entries + final entries = httpRequests.map((e) { + final requestCookies = e.requestCookies.map((cookie) { + return { + NetworkEventKeys.name: cookie.name, + NetworkEventKeys.value: cookie.value, + 'path': cookie.path, + 'domain': cookie.domain, + 'expires': cookie.expires?.toUtc().toIso8601String(), + 'httpOnly': cookie.httpOnly, + 'secure': cookie.secure, + }; + }).toList(); + + final requestHeaders = e.requestHeaders?.entries.map((header) { + var value = header.value; + if (value is List) { + value = value.first; + } + return { + NetworkEventKeys.name: header.key, + NetworkEventKeys.value: value, + }; + }).toList(); + + final queryString = Uri.parse(e.uri).queryParameters.entries.map((param) { + return { + NetworkEventKeys.name: param.key, + NetworkEventKeys.value: param.value, + }; + }).toList(); + + final responseCookies = e.responseCookies.map((cookie) { + return { + NetworkEventKeys.name: cookie.name, + NetworkEventKeys.value: cookie.value, + 'path': cookie.path, + 'domain': cookie.domain, + 'expires': cookie.expires?.toUtc().toIso8601String(), + 'httpOnly': cookie.httpOnly, + 'secure': cookie.secure, + }; + }).toList(); + + final responseHeaders = e.responseHeaders?.entries.map((header) { + var value = header.value; + if (value is List) { + value = value.first; + } + return { + NetworkEventKeys.name: header.key, + NetworkEventKeys.value: value, + }; + }).toList(); + + return { + NetworkEventKeys.startedDateTime: + e.startTimestamp.toUtc().toIso8601String(), + NetworkEventKeys.time: e.duration?.inMilliseconds, + // Request + NetworkEventKeys.request: { + NetworkEventKeys.method: e.method.toUpperCase(), + NetworkEventKeys.url: e.uri.toString(), + NetworkEventKeys.httpVersion: NetworkEventDefaults.httpVersion, + NetworkEventKeys.cookies: requestCookies, + NetworkEventKeys.headers: requestHeaders, + NetworkEventKeys.queryString: queryString, + NetworkEventKeys.postData: { + NetworkEventKeys.mimeType: e.contentType, + NetworkEventKeys.text: e.requestBody, + }, + NetworkEventKeys.headersSize: _calculateHeadersSize(e.requestHeaders), + NetworkEventKeys.bodySize: _calculateBodySize(e.requestBody), + }, + // Response + NetworkEventKeys.response: { + NetworkEventKeys.status: e.status, + NetworkEventKeys.statusText: '', + NetworkEventKeys.httpVersion: NetworkEventDefaults.responseHttpVersion, + NetworkEventKeys.cookies: responseCookies, + NetworkEventKeys.headers: responseHeaders, + NetworkEventKeys.content: { + NetworkEventKeys.size: e.responseBody?.length, + NetworkEventKeys.mimeType: e.type, + NetworkEventKeys.text: e.responseBody, + }, + NetworkEventKeys.redirectURL: '', + NetworkEventKeys.headersSize: _calculateHeadersSize(e.responseHeaders), + NetworkEventKeys.bodySize: _calculateBodySize(e.responseBody), + }, + // Cache + NetworkEventKeys.cache: {}, + NetworkEventKeys.timings: { + NetworkEventKeys.blocked: NetworkEventDefaults.blocked, + NetworkEventKeys.dns: NetworkEventDefaults.dns, + NetworkEventKeys.connect: NetworkEventDefaults.connect, + NetworkEventKeys.send: NetworkEventDefaults.send, + NetworkEventKeys.wait: e.duration!.inMilliseconds - 2, + NetworkEventKeys.receive: NetworkEventDefaults.receive, + NetworkEventKeys.ssl: NetworkEventDefaults.ssl, + }, + NetworkEventKeys.connection: e.hashCode.toString(), + NetworkEventKeys.comment: '', + }; + }).toList(); + + // Assemble the final HAR object + return { + NetworkEventKeys.log: { + NetworkEventKeys.version: NetworkEventDefaults.logVersion, + NetworkEventKeys.creator: creator, + NetworkEventKeys.entries: entries, + }, + }; +} + +int _calculateHeadersSize(Map? headers) { + if (headers == null) return -1; + + // Combine headers into a single string with CRLF endings + String headersString = headers.entries.map((entry) { + final key = entry.key; + var value = entry.value; + // If the value is a List, join it with a comma + if (value is List) { + value = value.join(', '); + } + return '$key: $value\r\n'; + }).join(); + + // Add final CRLF to indicate end of headers + headersString += '\r\n'; + + // Calculate the byte length of the headers string + return utf8.encode(headersString).length; +} + +int _calculateBodySize(String? requestBody) { + if (requestBody == null || requestBody.isEmpty) { + return 0; + } + return utf8.encode(requestBody).length; +} diff --git a/packages/devtools_app/lib/src/screens/network/network_controller.dart b/packages/devtools_app/lib/src/screens/network/network_controller.dart index c623f74cd19..973537123bf 100644 --- a/packages/devtools_app/lib/src/screens/network/network_controller.dart +++ b/packages/devtools_app/lib/src/screens/network/network_controller.dart @@ -3,19 +3,23 @@ // found in the LICENSE file. import 'dart:async'; +import 'dart:convert'; import 'package:devtools_app_shared/utils.dart'; import 'package:flutter/foundation.dart'; import 'package:vm_service/vm_service.dart'; +import '../../shared/config_specific/import_export/import_export.dart'; import '../../shared/config_specific/logger/allowed_error.dart'; import '../../shared/globals.dart'; import '../../shared/http/http_request_data.dart'; import '../../shared/http/http_service.dart' as http_service; +import '../../shared/offline_data.dart'; import '../../shared/primitives/utils.dart'; import '../../shared/ui/filter.dart'; import '../../shared/ui/search.dart'; import '../../shared/utils.dart'; +import 'har_builder.dart'; import 'network_model.dart'; import 'network_screen.dart'; import 'network_service.dart'; @@ -46,6 +50,7 @@ class NetworkController extends DisposableController with SearchControllerMixin, FilterControllerMixin, + OfflineScreenControllerMixin, AutoDisposeControllerMixin { NetworkController() { _networkService = NetworkService(this); @@ -56,6 +61,33 @@ class NetworkController extends DisposableController ); subscribeToFilterChanges(); } + List? httpRequests; + + String? exportAsHarFile() { + final exportController = ExportController(); + httpRequests = + filteredData.value.whereType().toList(); + + if (httpRequests!.isEmpty) { + debugPrint('No valid request data to export'); + return ''; + } + + try { + if (httpRequests != null && httpRequests!.isNotEmpty) { + // Build the HAR object + final har = buildHar(httpRequests!); + debugPrint('data is ${json.encode(har)}'); + return exportController.downloadFile( + json.encode(har), + type: ExportFileType.har, + ); + } + } catch (ex) { + debugPrint('Exception in export $ex'); + } + return null; + } static const methodFilterId = 'network-method-filter'; @@ -360,6 +392,25 @@ class NetworkController extends DisposableController serviceConnection.errorBadgeManager.incrementBadgeCount(NetworkScreen.id); } } + + @override + OfflineScreenData prepareOfflineScreenData() { + final requests = + filteredData.value.whereType().toList(); + return OfflineScreenData( + screenId: NetworkScreen.id, + data: convertRequestsToMap(requests), + ); + } +} + +Map convertRequestsToMap( + List? requests, +) { + if (requests == null) return {}; + return { + 'requests': requests.map((request) => request.toMap()).toList(), + }; } /// Class for managing the set of all current websocket requests, and diff --git a/packages/devtools_app/lib/src/screens/network/network_screen.dart b/packages/devtools_app/lib/src/screens/network/network_screen.dart index 92bab290b03..cd4aa5e423d 100644 --- a/packages/devtools_app/lib/src/screens/network/network_screen.dart +++ b/packages/devtools_app/lib/src/screens/network/network_screen.dart @@ -10,24 +10,16 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:provider/provider.dart'; +import '../../../devtools_app.dart'; import '../../shared/analytics/analytics.dart' as ga; import '../../shared/analytics/constants.dart' as gac; -import '../../shared/common_widgets.dart'; import '../../shared/config_specific/copy_to_clipboard/copy_to_clipboard.dart'; -import '../../shared/globals.dart'; import '../../shared/http/curl_command.dart'; -import '../../shared/http/http_request_data.dart'; -import '../../shared/primitives/utils.dart'; -import '../../shared/screen.dart'; import '../../shared/table/table.dart'; import '../../shared/table/table_data.dart'; -import '../../shared/ui/filter.dart'; -import '../../shared/ui/search.dart'; import '../../shared/ui/utils.dart'; -import '../../shared/utils.dart'; -import 'network_controller.dart'; -import 'network_model.dart'; import 'network_request_inspector.dart'; +import 'network_singleton.dart'; class NetworkScreen extends Screen { NetworkScreen() : super.fromMetaData(ScreenMetaData.network); @@ -116,6 +108,7 @@ class _NetworkScreenBodyState extends State @override void initState() { super.initState(); + addAutoDisposeListener(offlineDataController.showingOfflineData); ga.screen(NetworkScreen.id); } @@ -123,19 +116,41 @@ class _NetworkScreenBodyState extends State void didChangeDependencies() { super.didChangeDependencies(); if (!initController()) return; - unawaited(controller.startRecording()); - - cancelListeners(); + try { + if (offlineDataController.showingOfflineData.value == true) { + loadOfflineData(offlineDataController.offlineDataJson); + } + if (!offlineDataController.showingOfflineData.value) { + debugPrint('started recording'); + unawaited(controller.startRecording()); + } + cancelListeners(); + if (!offlineDataController.showingOfflineData.value) { + debugPrint('started recording'); + addAutoDisposeListener( + serviceConnection.serviceManager.isolateManager.mainIsolate, + () { + if (serviceConnection + .serviceManager.isolateManager.mainIsolate.value != + null) { + unawaited(controller.startRecording()); + } + }, + ); + } + } catch (ex) { + debugPrint('caught ex $ex'); + } + } - addAutoDisposeListener( - serviceConnection.serviceManager.isolateManager.mainIsolate, - () { - if (serviceConnection.serviceManager.isolateManager.mainIsolate.value != - null) { - unawaited(controller.startRecording()); - } - }, - ); + void loadOfflineData(Map offlineData) { + final requestsMap = offlineData['network']['requests'] as List; + final requests = requestsMap + .map((e) => DartIOHttpRequestData.fromJson(e as Map)) + .toList(); + controller.filteredData + ..clear() + ..addAll(requests); } @override @@ -143,7 +158,10 @@ class _NetworkScreenBodyState extends State // TODO(kenz): this won't work well if we eventually have multiple clients // that want to listen to network data. super.dispose(); - unawaited(controller.stopRecording()); + if (!(NetworkSingleton.instance.offlineMode ?? false)) { + debugPrint('stopped recording'); + unawaited(controller.stopRecording()); + } } @override @@ -200,10 +218,12 @@ class _NetworkProfilerControlsState extends State<_NetworkProfilerControls> _requests = widget.controller.requests.value; }); }); - _filteredRequests = widget.controller.filteredData.value; + NetworkSingleton.instance.filteredData = + widget.controller.filteredData.value; addAutoDisposeListener(widget.controller.filteredData, () { setState(() { - _filteredRequests = widget.controller.filteredData.value; + NetworkSingleton.instance.filteredData = + widget.controller.filteredData.value; }); }); } @@ -211,7 +231,7 @@ class _NetworkProfilerControlsState extends State<_NetworkProfilerControls> @override Widget build(BuildContext context) { final screenWidth = ScreenSize(context).width; - final hasRequests = _filteredRequests.isNotEmpty; + final hasRequests = NetworkSingleton.instance.filteredData?.isNotEmpty; return Row( children: [ StartStopRecordingButton( @@ -234,11 +254,20 @@ class _NetworkProfilerControlsState extends State<_NetworkProfilerControls> onPressed: widget.controller.clear, ), const SizedBox(width: defaultSpacing), + if (!offlineDataController.showingOfflineData.value) + DownloadButton( + minScreenWidthForTextBeforeScaling: + _NetworkProfilerControls._includeTextWidth, + onPressed: widget.controller.exportAsHarFile, + gaScreen: gac.network, + gaSelection: gac.NetworkEvent.networkDownloadHar, + ), + const SizedBox(width: defaultSpacing), const Expanded(child: SizedBox()), // TODO(kenz): fix focus issue when state is refreshed SearchField( searchController: widget.controller, - searchFieldEnabled: hasRequests, + searchFieldEnabled: hasRequests ?? false, searchFieldWidth: screenWidth <= MediaSize.xs ? defaultSearchFieldWidth : wideSearchFieldWidth, @@ -246,7 +275,8 @@ class _NetworkProfilerControlsState extends State<_NetworkProfilerControls> const SizedBox(width: denseSpacing), DevToolsFilterButton( onPressed: _showFilterDialog, - isFilterActive: _filteredRequests.length != _requests.length, + isFilterActive: NetworkSingleton.instance.filteredData?.length != + _requests.length, ), ], ); diff --git a/packages/devtools_app/lib/src/shared/analytics/constants.dart b/packages/devtools_app/lib/src/shared/analytics/constants.dart index f3830805d7b..7b7b1fbd144 100644 --- a/packages/devtools_app/lib/src/shared/analytics/constants.dart +++ b/packages/devtools_app/lib/src/shared/analytics/constants.dart @@ -12,6 +12,7 @@ part 'constants/_debugger_constants.dart'; part 'constants/_deep_links_constants.dart'; part 'constants/_extension_constants.dart'; part 'constants/_memory_constants.dart'; +part 'constants/_network_constants.dart'; part 'constants/_logging_constants.dart'; part 'constants/_performance_constants.dart'; part 'constants/_vs_code_sidebar_constants.dart'; diff --git a/packages/devtools_app/lib/src/shared/analytics/constants/_network_constants.dart b/packages/devtools_app/lib/src/shared/analytics/constants/_network_constants.dart new file mode 100644 index 00000000000..5c66632d6f2 --- /dev/null +++ b/packages/devtools_app/lib/src/shared/analytics/constants/_network_constants.dart @@ -0,0 +1,73 @@ +// Copyright 2024 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +part of '../constants.dart'; + +class NetworkEvent { + static const networkDownloadHar = 'networkDownloadHar'; +} + +class NetworkEventKeys { + static const log = 'log'; + static const version = 'version'; + static const creator = 'creator'; + static const name = 'name'; + static const creatorVersion = 'version'; + static const pages = 'pages'; + static const startedDateTime = 'startedDateTime'; + static const id = 'id'; + static const title = 'title'; + static const pageTimings = 'pageTimings'; + static const onContentLoad = 'onContentLoad'; + static const onLoad = 'onLoad'; + static const entries = 'entries'; + static const pageref = 'pageref'; + static const time = 'time'; + static const request = 'request'; + static const method = 'method'; + static const url = 'url'; + static const httpVersion = 'httpVersion'; + static const cookies = 'cookies'; + static const headers = 'headers'; + static const queryString = 'queryString'; + static const postData = 'postData'; + static const mimeType = 'mimeType'; + static const text = 'text'; + static const headersSize = 'headersSize'; + static const bodySize = 'bodySize'; + static const response = 'response'; + static const status = 'status'; + static const statusText = 'statusText'; + static const content = 'content'; + static const size = 'size'; + static const redirectURL = 'redirectURL'; + static const cache = 'cache'; + static const timings = 'timings'; + static const blocked = 'blocked'; + static const dns = 'dns'; + static const connect = 'connect'; + static const send = 'send'; + static const wait = 'wait'; + static const receive = 'receive'; + static const ssl = 'ssl'; + static const serverIPAddress = 'serverIPAddress'; + static const connection = 'connection'; + static const comment = 'comment'; + static const value = 'value'; +} + +class NetworkEventDefaults { + static const logVersion = '1.2'; + static const creatorName = 'devtools'; + static const onContentLoad = -1; + static const onLoad = -1; + static const httpVersion = 'HTTP/1.1'; + static const responseHttpVersion = 'http/2.0'; + static const blocked = -1; + static const dns = -1; + static const connect = -1; + static const send = 1; + static const receive = 1; + static const ssl = -1; +} diff --git a/packages/devtools_app/lib/src/shared/config_specific/import_export/import_export.dart b/packages/devtools_app/lib/src/shared/config_specific/import_export/import_export.dart index 903c12d03cb..356db3f61da 100644 --- a/packages/devtools_app/lib/src/shared/config_specific/import_export/import_export.dart +++ b/packages/devtools_app/lib/src/shared/config_specific/import_export/import_export.dart @@ -113,7 +113,8 @@ enum ExportFileType { json, csv, yaml, - data; + data, + har; @override String toString() => name; diff --git a/packages/devtools_app/lib/src/shared/feature_flags.dart b/packages/devtools_app/lib/src/shared/feature_flags.dart index 41ca89d482c..6976578862f 100644 --- a/packages/devtools_app/lib/src/shared/feature_flags.dart +++ b/packages/devtools_app/lib/src/shared/feature_flags.dart @@ -38,6 +38,9 @@ bool get enableBeta => enableExperiments || !isExternalBuild; const _kMemoryOfflineExperiment = bool.fromEnvironment('memory_offline_experiment'); +const _kNetworkOfflineExperiment = + bool.fromEnvironment('network_offline_experiment'); + // It is ok to have enum-like static only classes. // ignore: avoid_classes_with_only_static_members /// Flags to hide features under construction. @@ -62,6 +65,12 @@ abstract class FeatureFlags { static const memoryOffline = _kMemoryOfflineExperiment; // requires special handling because it needs to be const + /// Flag to enable offline data on network screen. + /// + /// https://github.com/flutter/devtools/issues/3806 + static const networkOffline = + _kNetworkOfflineExperiment; // requires special handling because it needs to be const + /// Flag to enable the deep link validation tooling in DevTools, both for the /// DevTools screen and the standalone tool for IDE embedding. /// diff --git a/packages/devtools_app/lib/src/shared/http/http_request_data.dart b/packages/devtools_app/lib/src/shared/http/http_request_data.dart index 1196793b4d7..b13e412f5cf 100644 --- a/packages/devtools_app/lib/src/shared/http/http_request_data.dart +++ b/packages/devtools_app/lib/src/shared/http/http_request_data.dart @@ -19,6 +19,17 @@ final _log = Logger('http_request_data'); /// Used to represent an instant event emitted during an HTTP request. class DartIOHttpInstantEvent { + factory DartIOHttpInstantEvent.fromJson(Map json) { + final event = HttpProfileRequestEvent( + event: json['event'] ?? '', + timestamp: DateTime.parse(json['timestamp']), + // Populate other necessary fields + ); + + final instantEvent = DartIOHttpInstantEvent._(event); + instantEvent._timeRange = TimeRange.fromJson(json['timeRange']); + return instantEvent; + } DartIOHttpInstantEvent._(this._event); final HttpProfileRequestEvent _event; @@ -37,6 +48,26 @@ class DartIOHttpInstantEvent { /// An abstraction of an HTTP request made through dart:io. class DartIOHttpRequestData extends NetworkRequest { + factory DartIOHttpRequestData.fromJson(Map json) { + final request = HttpProfileRequestRef( + id: json['id'], + method: json['method'], + uri: Uri.parse(json['uri']), + isolateId: '123', + events: [], + startTime: DateTime.now(), + ); + + final data = DartIOHttpRequestData(request); + data._responseBody = json['responseBody'] ?? ''; + data._requestBody = json['requestBody'] ?? ''; + // data._instantEvents = (json['instantEvents'] as List) + // .map((e) => DartIOHttpInstantEvent.fromJson(e as Map)) + // .toList(); + + // Populate other fields as needed + return data; + } DartIOHttpRequestData( this._request, { bool requestFullDataFromVmService = true, @@ -319,4 +350,27 @@ class DartIOHttpRequestData extends NetworkRequest { port, startTimestamp, ); + + Map toMap() { + return { + 'id': id, + 'method': method, + 'uri': uri, + 'status': 'status', + 'type': type, + 'duration': 'duration?.inMilliseconds', + 'startTimestamp': startTimestamp.toIso8601String(), + 'endTimestamp': endTimestamp!, + 'requestHeaders': requestHeaders ?? {}, + 'responseHeaders': responseHeaders ?? {}, + 'requestBody': requestBody ?? '', + 'responseBody': responseBody ?? '', + 'inProgress': inProgress, + 'didFail': didFail, + 'general': general, + 'requestCookies': requestCookies.map((c) => c.toString()).toList(), + 'responseCookies': responseCookies.map((c) => c.toString()).toList(), + 'instantEvents': 'instantEvents.map((e) => ()).toList()', + }; + } } diff --git a/packages/devtools_app/lib/src/shared/primitives/utils.dart b/packages/devtools_app/lib/src/shared/primitives/utils.dart index b5e453c47f7..488b355a790 100644 --- a/packages/devtools_app/lib/src/shared/primitives/utils.dart +++ b/packages/devtools_app/lib/src/shared/primitives/utils.dart @@ -454,6 +454,12 @@ enum TimeUnit { class TimeRange { TimeRange({this.singleAssignment = true}); + factory TimeRange.fromJson(Map json) { + return TimeRange( + singleAssignment: true, + ); + } + factory TimeRange.offset({ required TimeRange original, required Duration offset, diff --git a/packages/devtools_app/lib/src/shared/screen.dart b/packages/devtools_app/lib/src/shared/screen.dart index 3e17e38ded5..f63427a87b1 100644 --- a/packages/devtools_app/lib/src/shared/screen.dart +++ b/packages/devtools_app/lib/src/shared/screen.dart @@ -75,6 +75,8 @@ enum ScreenMetaData { icon: Icons.network_check, requiresDartVm: true, tutorialVideoTimestamp: '?t=547', + // ignore: avoid_redundant_argument_values, false positive + worksWithOfflineData: FeatureFlags.networkOffline, ), logging( 'logging',