-
Notifications
You must be signed in to change notification settings - Fork 189
Added cf-profile.py script for processing profiling output #5915
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
victormlg
wants to merge
2
commits into
cfengine:master
Choose a base branch
from
victormlg:cf_profile_py
base: master
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
2 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,153 @@ | ||
| from argparse import ArgumentParser | ||
| import sys | ||
| import json | ||
| import re | ||
|
||
| from collections import defaultdict | ||
|
|
||
| """ | ||
| This script parses profiling output from cf-agent and lists policy bundles, promises, and functions sorted by execution time. | ||
|
|
||
| Usage: | ||
| $ sudo /var/cfengine/cf-agent -Kp | ||
| $ python3 cf-profile.py profiling_output.json --bundles --promises --functions | ||
| """ | ||
|
|
||
victormlg marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
|
||
| def parse_args(): | ||
| parser = ArgumentParser() | ||
|
|
||
| # parser.add_argument("profiling_output") | ||
| parser.add_argument("--top", type=int, default=10) | ||
| parser.add_argument("--bundles", action="store_true") | ||
| parser.add_argument("--promises", action="store_true") | ||
| parser.add_argument("--functions", action="store_true") | ||
| parser.add_argument("--flamegraph", action="store_true") | ||
|
|
||
| return parser.parse_args() | ||
|
|
||
|
|
||
| def format_elapsed_time(elapsed_ns): | ||
| elapsed_ms = float(elapsed_ns) / 1e6 | ||
|
|
||
| if elapsed_ms < 1000: | ||
| return "%.2f ms" % elapsed_ms | ||
| elif elapsed_ms < 60000: | ||
| elapsed_s = elapsed_ms / 1000.0 | ||
| return "%.2fs" % elapsed_s | ||
| else: | ||
| elapsed_s = elapsed_ms / 1000.0 | ||
| minutes = int(elapsed_s // 60) | ||
| seconds = int(elapsed_s % 60) | ||
| return "%dm%ds" % (minutes, seconds) | ||
|
|
||
|
|
||
| def format_label(event_type, name): | ||
| if event_type == "function": | ||
| return "%s() %s call" % (name, event_type) | ||
| elif name == "methods": | ||
| return "bundle invocation" | ||
| else: | ||
| return "%s %s" % (name, event_type) | ||
|
|
||
|
|
||
| def format_columns(events, top): | ||
|
|
||
| labels = [] | ||
|
|
||
| for event in events[:top]: | ||
| label = format_label(event["type"], event["name"]) | ||
| location = "%s:%s" % (event["filename"], event["offset"]["line"]) | ||
| time = format_elapsed_time(event["elapsed"]) | ||
|
|
||
| labels.append((label, location, time)) | ||
|
|
||
| return labels | ||
|
|
||
|
|
||
| def get_max_column_lengths(lines, indent=4): | ||
|
|
||
| max_type, max_location, max_time = 0, 0, 0 | ||
|
|
||
| for label, location, time_ms in lines: | ||
| max_type = max(max_type, len(label)) | ||
| max_location = max(max_location, len(location)) | ||
| max_time = max(max_time, len(time_ms)) | ||
|
|
||
| return max_type + indent, max_location + indent, max_time + indent | ||
|
|
||
|
|
||
| def profile(data, args): | ||
|
|
||
| events = data["events"] | ||
| filter = defaultdict(list) | ||
|
|
||
| if args.bundles: | ||
| filter["type"].append("bundle") | ||
| filter["name"].append("methods") | ||
|
|
||
| if args.promises: | ||
| filter["name"] += list( | ||
| set( | ||
| event["name"] | ||
| for event in events | ||
| if event["type"] == "promise" and event["name"] != "methods" | ||
| ) | ||
| ) | ||
|
|
||
| if args.functions: | ||
| filter["type"].append("function") | ||
|
|
||
| # filter events | ||
| if filter is not None: | ||
| events = [ | ||
| event | ||
| for field in filter.keys() | ||
| for event in events | ||
| if event[field] in filter[field] | ||
| ] | ||
|
|
||
| # sort events | ||
| events = sorted(events, key=lambda x: x["elapsed"], reverse=True) | ||
|
|
||
| lines = format_columns(events, args.top) | ||
| line_format = "%-{}s %-{}s %{}s".format(*get_max_column_lengths(lines)) | ||
|
|
||
| # print top k filtered events | ||
| print(line_format % ("Type", "Location", "Time")) | ||
| for label, location, time_ms in lines: | ||
| print(line_format % (label, location, time_ms)) | ||
|
|
||
| def generate_callstacks(data): | ||
| events = data["events"] | ||
|
|
||
| # hashmap = {"policy" : 0} | ||
| # for event in events: | ||
| # hashmap[event["id"]] = event["elapsed"] | ||
|
Comment on lines
+124
to
+125
Check noticeCode scanning / CodeQL Commented-out code Note
This comment appears to contain commented-out code.
|
||
| # print(json.dumps(hashmap, indent=4)) | ||
|
|
||
| # for event in sorted(events, key=lambda x: x["id"].count(";"), reverse=True): | ||
| # parent = ";".join(event["id"].split(";")[:-1]) | ||
| # hashmap[parent] -= event["elapsed"] | ||
|
Comment on lines
+128
to
+130
Check noticeCode scanning / CodeQL Commented-out code Note
This comment appears to contain commented-out code.
|
||
|
|
||
|
|
||
| with open("stack.txt", "w") as f: | ||
| for event in events: | ||
| f.write("%s %d\n" % (event["callstack"], event["elapsed"])) | ||
|
|
||
| def main(): | ||
| args = parse_args() | ||
| m = re.search(r"\{[.\s\S]*\}", sys.stdin.read()) | ||
| # with open(args.profiling_output, "r") as f: | ||
| # data = json.load(f) | ||
|
Comment on lines
+140
to
+141
Check noticeCode scanning / CodeQL Commented-out code Note
This comment appears to contain commented-out code.
|
||
| data = json.loads(m.group(0)) | ||
|
|
||
|
|
||
| if args.flamegraph: | ||
| generate_callstacks(data) | ||
| else: | ||
| profile(data, args) | ||
|
|
||
|
|
||
|
|
||
| if __name__ == "__main__": | ||
| main() | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.