Skip to content

Commit bae4b27

Browse files
rahul2393rahul2393
authored
Merge branch 'main' into eef
2 parents a3bb4dc + f1faf60 commit bae4b27

5,511 files changed

Lines changed: 711955 additions & 40370 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/release-note-generation/generate_module_notes.py‎

Lines changed: 335 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,335 @@
1+
import argparse
2+
import re
3+
import subprocess
4+
import sys
5+
6+
7+
def run_cmd(cmd, cwd=None):
8+
"""Runs a shell command and returns the output."""
9+
result = subprocess.run(
10+
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, cwd=cwd
11+
)
12+
if result.returncode != 0:
13+
print(f"Error running command: {' '.join(cmd)}", file=sys.stderr)
14+
print(result.stderr, file=sys.stderr)
15+
sys.exit(result.returncode)
16+
return result.stdout
17+
18+
19+
def find_version_boundaries(file_path, pattern, target_version, module=None):
20+
"""Scans history of a file to find release boundaries moving forward."""
21+
log_cmd = [
22+
"git",
23+
"log",
24+
"--oneline",
25+
"--all",
26+
"--",
27+
file_path,
28+
]
29+
try:
30+
log_output = run_cmd(log_cmd)
31+
commits = [line.split()[0] for line in log_output.splitlines() if line]
32+
commits.reverse() # Move forward in time!
33+
34+
first_prev_commit = None
35+
target_release_commit = None
36+
prev_version = None
37+
38+
for commit in commits:
39+
# Check if file exists at that commit to avoid noisy errors
40+
check_cmd = ["git", "cat-file", "-e", f"{commit}:{file_path}"]
41+
check_result = subprocess.run(check_cmd, stderr=subprocess.PIPE)
42+
if check_result.returncode != 0:
43+
continue
44+
45+
show_cmd = ["git", "show", f"{commit}:{file_path}"]
46+
try:
47+
content = run_cmd(show_cmd)
48+
except SystemExit:
49+
continue
50+
51+
found_ver = None
52+
match = pattern.search(content)
53+
if match:
54+
found_ver = match.group(1)
55+
56+
if found_ver:
57+
if found_ver == target_version:
58+
target_release_commit = commit
59+
break # Stop as soon as we find the target release!
60+
61+
# Track the first occurrence of the latest stable version before target
62+
if found_ver != target_version and "-SNAPSHOT" not in found_ver and (not prev_version or found_ver != prev_version):
63+
prev_version = found_ver
64+
first_prev_commit = commit
65+
66+
return first_prev_commit, target_release_commit, prev_version
67+
except SystemExit:
68+
return None, None, None
69+
70+
71+
def verify_commit(commit_hash, directory, module, allowed_versions):
72+
"""Verifies if a commit belongs to the release based on file state."""
73+
if directory == ".":
74+
pom_path = "gapic-libraries-bom/pom.xml"
75+
else:
76+
pom_path = f"{directory}/pom.xml"
77+
78+
# Check if file exists at that commit to avoid noisy errors
79+
check_cmd = ["git", "cat-file", "-e", f"{commit_hash}:{pom_path}"]
80+
check_result = subprocess.run(check_cmd, stderr=subprocess.PIPE)
81+
if check_result.returncode != 0:
82+
return False
83+
84+
try:
85+
content = run_cmd(["git", "show", f"{commit_hash}:{pom_path}"])
86+
# Allow optional <packaging> tag in between artifactId and version
87+
pattern = re.compile(rf"<artifactId>{re.escape(module)}</artifactId>\s*(?:<packaging>[^<]+</packaging>\s*)?<version>([^<]+)</version>", re.DOTALL)
88+
89+
match = pattern.search(content)
90+
if match and match.group(1) in allowed_versions:
91+
return True
92+
except SystemExit:
93+
pass
94+
95+
return False
96+
97+
98+
def parse_commit_overrides(commit_data, short_name, prefix_regex, commit_hash, categorize_callback):
99+
"""Parses commit overrides and calls callback for each item."""
100+
match = re.search(r"BEGIN_COMMIT_OVERRIDE(.*?)END_COMMIT_OVERRIDE", commit_data, re.DOTALL)
101+
if not match:
102+
return False
103+
104+
override_content = match.group(1)
105+
current_item = []
106+
in_module_item = False
107+
108+
for line in override_content.splitlines():
109+
line_stripped = line.strip()
110+
if not line_stripped:
111+
continue
112+
113+
is_new_item = prefix_regex.match(line_stripped)
114+
115+
if is_new_item:
116+
if in_module_item and current_item:
117+
categorize_callback(commit_hash, " ".join(current_item))
118+
current_item = []
119+
in_module_item = False
120+
121+
should_include = False
122+
if short_name:
123+
if f"[{short_name}]" in line_stripped or f"({short_name})" in line_stripped:
124+
should_include = True
125+
else:
126+
should_include = True
127+
128+
if should_include:
129+
in_module_item = True
130+
current_item.append(line_stripped)
131+
elif in_module_item:
132+
if line_stripped.startswith(("PiperOrigin-RevId:", "Source Link:")):
133+
continue
134+
if line_stripped in ("END_NESTED_COMMIT", "BEGIN_NESTED_COMMIT"):
135+
continue
136+
current_item.append(line_stripped)
137+
138+
if in_module_item and current_item:
139+
categorize_callback(commit_hash, " ".join(current_item))
140+
141+
return True
142+
143+
144+
def get_tag_or_commit(commit_hash, target_version):
145+
"""Returns the tag pointing at the commit if there is exactly one, else the commit hash."""
146+
if not commit_hash:
147+
return None
148+
try:
149+
# Remove ~1 if present to find the actual tag pointing at the commit
150+
clean_hash = commit_hash.split("~")[0]
151+
tags_output = run_cmd(["git", "tag", "--points-at", clean_hash])
152+
tags = [line.strip() for line in tags_output.splitlines() if line.strip()]
153+
if len(tags) == 1:
154+
return tags[0]
155+
elif len(tags) > 1:
156+
for tag in tags:
157+
if target_version in tag:
158+
return tag
159+
except SystemExit:
160+
pass
161+
return commit_hash
162+
163+
164+
def main():
165+
parser = argparse.ArgumentParser(
166+
description="Generate release notes based on commit history for a specific module."
167+
)
168+
parser.add_argument(
169+
"--module", required=True, help="Module name as specified in versions.txt"
170+
)
171+
parser.add_argument(
172+
"--directory", required=True, help="Path in the monorepo where the module has code"
173+
)
174+
parser.add_argument("--version", required=True, help="Target version")
175+
parser.add_argument(
176+
"--short-name", help="Module short-name used in commit overrides (e.g., aiplatform). Omit for repo-wide generation."
177+
)
178+
args = parser.parse_args()
179+
180+
module = args.module
181+
directory = args.directory
182+
target_version = args.version
183+
184+
# 1. Scan history of pom.xml
185+
if directory == ".":
186+
pom_path = "gapic-libraries-bom/pom.xml"
187+
else:
188+
pom_path = f"{directory}/pom.xml"
189+
pom_pattern = re.compile(r"<version>([^<]+)</version>")
190+
191+
prev_commit, target_release_commit, prev_version = find_version_boundaries(pom_path, pom_pattern, target_version)
192+
193+
target_commit = None
194+
if target_release_commit:
195+
target_commit = target_release_commit
196+
print(f"Found target release commit at {target_release_commit}. Using inclusive upper boundary {target_commit}", file=sys.stderr)
197+
198+
if not target_commit:
199+
print(f"Target version {target_version} not found in history of {pom_path}.", file=sys.stderr)
200+
sys.exit(1)
201+
202+
range_desc = f"between {prev_commit} and {target_commit}" if prev_commit else f"up to {target_commit}"
203+
print(
204+
f"Generating notes {range_desc} for directory {directory}", file=sys.stderr
205+
)
206+
207+
# 2. Generate commit history in that range affecting that directory
208+
# Use format that includes hash, subject, and body
209+
notes_cmd = [
210+
"git",
211+
"log",
212+
"--format=%H %s%n%b%n--END_OF_COMMIT--",
213+
f"{prev_commit}~1..{target_commit}" if prev_commit else target_commit,
214+
]
215+
if directory != ".":
216+
notes_cmd.extend(["--", directory])
217+
notes_output = run_cmd(notes_cmd)
218+
219+
220+
221+
# Filter commit titles based on allowed prefixes and categorize them
222+
# Supports scopes in parentheses, e.g., feat(spanner):
223+
prefix_regex = re.compile(r"^(feat|fix|deps|docs|chore\(deps\)|build\(deps\))(\([^)]+\))?(!)?:")
224+
225+
breaking_changes = []
226+
features = []
227+
bug_fixes = []
228+
dependency_upgrades = []
229+
documentation = []
230+
231+
def categorize_and_append(commit_hash, text):
232+
match = prefix_regex.match(text)
233+
if not match:
234+
return
235+
236+
prefix = match.group(1)
237+
is_breaking = match.group(3) == "!"
238+
239+
commit_link = f"([{commit_hash[:7]}](https://github.com/googleapis/google-cloud-java/commit/{commit_hash}))"
240+
full_item = f"{text} {commit_link}"
241+
242+
if is_breaking:
243+
breaking_changes.append(full_item)
244+
elif prefix == "feat":
245+
features.append(full_item)
246+
elif prefix == "fix":
247+
bug_fixes.append(full_item)
248+
elif prefix == "deps" or prefix in ("chore(deps)", "build(deps)"):
249+
dependency_upgrades.append(full_item)
250+
elif prefix == "docs":
251+
documentation.append(full_item)
252+
253+
commits_data = notes_output.split("--END_OF_COMMIT--")
254+
255+
for commit_data in commits_data:
256+
commit_data = commit_data.strip()
257+
if not commit_data:
258+
continue
259+
260+
lines = commit_data.splitlines()
261+
if not lines:
262+
continue
263+
264+
header_parts = lines[0].split(" ", 1)
265+
commit_hash = header_parts[0]
266+
subject = header_parts[1] if len(header_parts) > 1 else ""
267+
268+
body = "\n".join(lines[1:])
269+
270+
# Verify if commit belongs to this release based on file state
271+
target_snapshot = f"{target_version}-SNAPSHOT"
272+
allowed_versions = (prev_version, target_snapshot) if prev_version else (target_snapshot,)
273+
274+
target_module = "gapic-libraries-bom" if directory == "." else module
275+
if not verify_commit(commit_hash, directory, target_module, allowed_versions):
276+
continue
277+
278+
# Check for override in the entire message
279+
if "BEGIN_COMMIT_OVERRIDE" in body or "BEGIN_COMMIT_OVERRIDE" in subject:
280+
if parse_commit_overrides(commit_data, args.short_name, prefix_regex, commit_hash, categorize_and_append):
281+
continue
282+
283+
# Fallback to title check if no override
284+
if prefix_regex.match(subject):
285+
categorize_and_append(commit_hash, subject)
286+
287+
# Get dates and build header
288+
target_date = run_cmd(["git", "log", "-1", "--format=%cI", target_commit]).strip()
289+
date_str = target_date.split("T")[0] # Get YYYY-MM-DD
290+
291+
prev_ref = get_tag_or_commit(prev_commit, prev_version) if prev_version else prev_commit
292+
target_ref = get_tag_or_commit(target_commit, target_version)
293+
294+
compare_url = f"https://github.com/googleapis/google-cloud-java/compare/{prev_ref}...{target_ref}" if prev_ref else f"https://github.com/googleapis/google-cloud-java/commit/{target_ref}"
295+
296+
print(f"## [{target_version}]({compare_url}) ({date_str})")
297+
print()
298+
299+
if not any([breaking_changes, features, bug_fixes, dependency_upgrades, documentation]):
300+
print("* No change")
301+
else:
302+
if breaking_changes:
303+
print("### ⚠ BREAKING CHANGES\n")
304+
for item in breaking_changes:
305+
print(f"* {item}")
306+
print()
307+
308+
if features:
309+
print("### Features\n")
310+
for item in features:
311+
print(f"* {item}")
312+
print()
313+
314+
if bug_fixes:
315+
print("### Bug Fixes\n")
316+
for item in bug_fixes:
317+
print(f"* {item}")
318+
print()
319+
320+
if documentation:
321+
print("### Documentation\n")
322+
for item in documentation:
323+
print(f"* {item}")
324+
print()
325+
326+
if dependency_upgrades:
327+
print("### Dependencies\n")
328+
for item in dependency_upgrades:
329+
print(f"* {item}")
330+
print()
331+
332+
333+
334+
if __name__ == "__main__":
335+
main()

‎.github/release-note-generation/split_release_note.py‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ def detect_modules(root_directory: Path):
6969
tree = ET.parse(module_pom_xml)
7070
root = tree.getroot()
7171
version = root.find('mvn:version', POM_NAMESPACES).text
72+
api_name = None
7273
if owlbot_yaml_path.exists():
7374
# If OwlBot configuration file exists (most cases), it's the better
7475
# source to get the OwlBot-generated pull request title prefix than
@@ -78,21 +79,21 @@ def detect_modules(root_directory: Path):
7879
match = re.search(r'api-name: (.+)', owlbot_yaml_content)
7980
if match:
8081
api_name = match.group(1)
81-
modules.append(LibraryModule(module_path, api_name,
82-
version,
83-
changelog))
82+
83+
if not api_name:
84+
# Fallback to repo-metadata.json (e.g. for vertexai or Spanner transitional state)
85+
if repo_metadata_path.exists():
86+
with open(repo_metadata_path, 'r') as file:
87+
repo_metadata = json.load(file)
88+
api_name = repo_metadata.get('api_shortname')
89+
90+
if api_name:
91+
modules.append(LibraryModule(module_path, api_name,
92+
version,
93+
changelog))
8494
else:
85-
# vertexai (handwritten) does not have OwlBot yaml file
86-
with open(repo_metadata_path, 'r') as file:
87-
repo_metadata = json.load(file)
88-
api_name = repo_metadata['api_shortname']
89-
if api_name:
90-
modules.append(LibraryModule(repo_metadata_path.parent, api_name,
91-
version,
92-
changelog))
93-
else:
94-
raise Exception(f'repo_metadata_path {repo_metadata_path} does'
95-
f' not have api_shortname field')
95+
raise Exception(f'Could not determine api-name for {repo_metadata_path}')
96+
9697

9798
return modules
9899

@@ -133,7 +134,7 @@ def group_changes_by_api(main_changes: [str]):
133134
elif section == BUG_FIXES_SECTION:
134135
api_to_changelog[api_name].bug_fixes.append(note)
135136
elif section == DEPENDENCIES_SECTION:
136-
api_to_changelog[api_name].dependencies.append(note)
137+
api_to_changelog[api_name].dependency_upgrades.append(note)
137138
return api_to_changelog
138139

139140

0 commit comments

Comments
 (0)