ENT-13808: Download releases.json directly to file and parse afterwards

Author: sarakthon

src/cfengine_cli/masterfiles/generate_release_information.py

@@ -1,4 +1,5 @@
 import json
+import requests
 from cfengine_cli.masterfiles.download import (
     ENTERPRISE_RELEASES_URL,
     COMMUNITY_RELEASES_URL,
@@ -53,10 +54,6 @@ def generate_release_information_impl(
     )
     generate_vcf_download(DOWNLOAD_PATH, downloaded_versions)
 
-    generate_release_history()
-
-    generate_git_tags_map()
-
     if check:
         print(
             "Downloading releases of masterfiles from git (github.com) and generating "
@@ -81,43 +78,37 @@ def generate_release_information_impl(
 def download_enterprise_releasedata():
     # Downloading releases.json:
     try:
-        releases_data = get_json(ENTERPRISE_RELEASES_URL)
+        return requests.get(ENTERPRISE_RELEASES_URL)
 
-    except CFBSNetworkError:
+    except requests.RequestException:
         raise CFBSExitError(
             "Downloading CFEngine release data failed - check your Wi-Fi / network settings."
         )
 
-    return releases_data
-
 
 def download_community_releasedata():
     # Downloading community/releases.json
     try:
-        releases_data = get_json(COMMUNITY_RELEASES_URL)
-
-    except CFBSNetworkError:
+        return requests.get(COMMUNITY_RELEASES_URL)
+    except requests.RequestException:
         raise CFBSExitError(
             "Downloading CFEngine release data failed - check your Wi-Fi / network settings."
         )
 
-    return releases_data
-
 
 def process_release_type(folder, download_func):
     # Function for processing either community or enterprise releases
-    release_data = download_func()
-
-    write_json_pretty(f"./{folder}/releases.json", release_data)
+    releases_path = f"./{folder}/releases.json"
+    print(f"\nDownloading release data for {folder}...")
+    response = download_func()
+    # Save raw file to preserve exact format from website
+    save_to_file(releases_path, response.content)
 
+    release_data = load_json_from_file(releases_path)
     stable_releases = get_stable_releases(release_data)
-
     file_checksums_dict = build_release_history(stable_releases)
-
     write_version_files(stable_releases, folder)
-
     sorted_releases = sort_release_data(file_checksums_dict)
-
     write_json(f"./{folder}/checksums.json", sorted_releases)
 
 
@@ -192,7 +183,21 @@ def write_version_files(stable_releases, folder):
         write_json(f"./{folder}/versions/{version}.json", version_data)
 
 
-def write_json_pretty(path, data):
-    # Writes release information in same format as on cfengine.com
-    with open(path, "w", encoding="utf-8") as f:
-        json.dump(data, f, indent=2, ensure_ascii=False)
+def save_to_file(path, content):
+    try:
+        with open(path, "wb") as f:
+            f.write(content)
+        print(f"Saved {path}")
+
+    except OSError:
+        raise CFBSExitError(f"Failed to write file {path}.")
+
+
+def load_json_from_file(path):
+    # Open saved file and return as dict
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+
+    except OSError:
+        raise CFBSExitError(f"Failed to read file {path}.")