From 8e53bfead6195e4440aef47b2f08b0e7f89b1d4d Mon Sep 17 00:00:00 2001 From: Nathan McCarty Date: Tue, 14 Jun 2022 00:55:47 -0400 Subject: [PATCH] Add semeru binaries --- flake.nix | 163 ++++++++++++++++++++++------------------ sources.json | 160 +++++++++++++++++++++++++++++---------- updater/src/adoptium.rs | 16 ++-- updater/src/main.rs | 46 ++++++++++-- updater/src/semeru.rs | 95 +++++++++++++++++++++++ 5 files changed, 351 insertions(+), 129 deletions(-) create mode 100644 updater/src/semeru.rs diff --git a/flake.nix b/flake.nix index 75320f8..4a98e96 100644 --- a/flake.nix +++ b/flake.nix @@ -10,76 +10,76 @@ utils.lib.eachSystem [ "x86_64-linux" ] (system: let sources = builtins.fromJSON (builtins.readFile ./sources.json); - buildAdoptLike = with import nixpkgs {system = system;}; name: value: - let - cpuName = stdenv.hostPlatform.parsed.cpu.name; - runtimeDependencies = [ - pkgs.cups - pkgs.cairo - pkgs.glib - pkgs.gtk3 - ]; - runtimeLibraryPath = lib.makeLibraryPath runtimeDependencies; - in - stdenv.mkDerivation rec { - name = "jdk${toString value.major_version}"; - src = builtins.fetchurl { - url = value.link; - sha256 = value.sha256; - }; - version = value.java_version; - buildInputs = with pkgs; [ - alsa-lib - fontconfig - freetype - stdenv.cc.cc.lib - xorg.libX11 - xorg.libXext - xorg.libXi - xorg.libXrender - xorg.libXtst - zlib - ]; - nativeBuildInputs = with pkgs; [ - autoPatchelfHook - makeWrapper - ]; - dontStrip = 1; - installPhase = '' - cd .. - mv $sourceRoot $out - # jni.h expects jni_md.h to be in the header search path. - ln -s $out/include/linux/*_md.h $out/include/ - rm -rf $out/demo - # Remove some broken manpages. - rm -rf $out/man/ja* - # Remove embedded freetype to avoid problems like - # https://github.com/NixOS/nixpkgs/issues/57733 - find "$out" -name 'libfreetype.so*' -delete - # Propagate the setJavaClassPath setup hook from the JDK so that - # any package that depends on the JDK has $CLASSPATH set up - # properly. - mkdir -p $out/nix-support - printWords ${setJavaClassPath} > $out/nix-support/propagated-build-inputs - # Set JAVA_HOME automatically. - cat <> "$out/nix-support/setup-hook" - if [ -z "\''${JAVA_HOME-}" ]; then export JAVA_HOME=$out; fi - EOF - # We cannot use -exec since wrapProgram is a function but not a command. - # - # jspawnhelper is executed from JVM, so it doesn't need to wrap it, and it - # breaks building OpenJDK (#114495). - for bin in $( find "$out" -executable -type f -not -name jspawnhelper ); do - if patchelf --print-interpreter "$bin" &> /dev/null; then - wrapProgram "$bin" --prefix LD_LIBRARY_PATH : "${runtimeLibraryPath}" - fi - done - ''; - preFixup = '' - find "$out" -name libfontmanager.so -exec \ - patchelf --add-needed libfontconfig.so {} \; - ''; + buildAdoptLike = with import nixpkgs { system = system; }; name: value: + let + cpuName = stdenv.hostPlatform.parsed.cpu.name; + runtimeDependencies = [ + pkgs.cups + pkgs.cairo + pkgs.glib + pkgs.gtk3 + ]; + runtimeLibraryPath = lib.makeLibraryPath runtimeDependencies; + in + stdenv.mkDerivation rec { + name = "jdk${toString value.major_version}"; + src = builtins.fetchurl { + url = value.link; + sha256 = value.sha256; }; + version = value.java_version; + buildInputs = with pkgs; [ + alsa-lib + fontconfig + freetype + stdenv.cc.cc.lib + xorg.libX11 + xorg.libXext + xorg.libXi + xorg.libXrender + xorg.libXtst + zlib + ]; + nativeBuildInputs = with pkgs; [ + autoPatchelfHook + makeWrapper + ]; + dontStrip = 1; + installPhase = '' + cd .. + mv $sourceRoot $out + # jni.h expects jni_md.h to be in the header search path. + ln -s $out/include/linux/*_md.h $out/include/ + rm -rf $out/demo + # Remove some broken manpages. + rm -rf $out/man/ja* + # Remove embedded freetype to avoid problems like + # https://github.com/NixOS/nixpkgs/issues/57733 + find "$out" -name 'libfreetype.so*' -delete + # Propagate the setJavaClassPath setup hook from the JDK so that + # any package that depends on the JDK has $CLASSPATH set up + # properly. + mkdir -p $out/nix-support + printWords ${setJavaClassPath} > $out/nix-support/propagated-build-inputs + # Set JAVA_HOME automatically. + cat <> "$out/nix-support/setup-hook" + if [ -z "\''${JAVA_HOME-}" ]; then export JAVA_HOME=$out; fi + EOF + # We cannot use -exec since wrapProgram is a function but not a command. + # + # jspawnhelper is executed from JVM, so it doesn't need to wrap it, and it + # breaks building OpenJDK (#114495). + for bin in $( find "$out" -executable -type f -not -name jspawnhelper ); do + if patchelf --print-interpreter "$bin" &> /dev/null; then + wrapProgram "$bin" --prefix LD_LIBRARY_PATH : "${runtimeLibraryPath}" + fi + done + ''; + preFixup = '' + find "$out" -name libfontmanager.so -exec \ + patchelf --add-needed libfontconfig.so {} \; + ''; + }; in with import nixpkgs { system = system; }; { @@ -88,15 +88,28 @@ buildAdoptLike name value ) sources.${system}.temurin.versions) // { - latest = buildAdoptLike "latest" sources.${system}.temurin.latest; - stable = buildAdoptLike "stable" sources.${system}.temurin.stable; - lts = buildAdoptLike "lts" sources.${system}.temurin.lts; - }; - + latest = buildAdoptLike "latest" sources.${system}.temurin.latest; + stable = buildAdoptLike "stable" sources.${system}.temurin.stable; + lts = buildAdoptLike "lts" sources.${system}.temurin.lts; + }; + packages.temurin-latest = self.packages.${system}.temurin.latest; packages.temurin-stable = self.packages.${system}.temurin.stable; packages.temurin-lts = self.packages.${system}.temurin.lts; - + + packages.semeru = (builtins.mapAttrs + (name: value: + buildAdoptLike name value) + sources.${system}.semeru.versions) // { + latest = buildAdoptLike "latest" sources.${system}.semeru.latest; + stable = buildAdoptLike "stable" sources.${system}.semeru.stable; + lts = buildAdoptLike "lts" sources.${system}.semeru.lts; + }; + + packages.semeru-latest = self.packages.${system}.semeru.latest; + packages.semeru-stable = self.packages.${system}.semeru.stable; + packages.semeru-lts = self.packages.${system}.semeru.lts; + defaultPackage = self.packages.${system}.stable; }); } diff --git a/sources.json b/sources.json index 1638ab4..45754bd 100644 --- a/sources.json +++ b/sources.json @@ -2,34 +2,6 @@ "x86_64-linux": { "temurin": { "versions": { - "jdk11": { - "link": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.15_10.tar.gz", - "major_version": 11, - "java_version": "11.0.15+10", - "early_access": false, - "sha256": "02sdnmk9zswi5skhyik1zaqm08v6cyggjc7c7ykwrw322rd4vnsz" - }, - "jdk17": { - "link": "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.3%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.3_7.tar.gz", - "major_version": 17, - "java_version": "17.0.3+7", - "early_access": false, - "sha256": "0wvbvv01jiwkj84kqnbnqfivjy0cca8h7d8996qgpybp239bxxc1" - }, - "jdk18": { - "link": "https://github.com/adoptium/temurin18-binaries/releases/download/jdk-18.0.1%2B10/OpenJDK18U-jdk_x64_linux_hotspot_18.0.1_10.tar.gz", - "major_version": 18, - "java_version": "18.0.1+10", - "early_access": false, - "sha256": "18mhq6bqhbig58w2n0iw2ssz9ir48dk9rzd10jpmgh92bzbxkc8n" - }, - "jdk19": { - "link": "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-2022-06-09-19-11-beta/OpenJDK-jdk_x64_linux_hotspot_2022-06-09-03-31.tar.gz", - "major_version": 19, - "java_version": "19-beta+25-202206090350", - "early_access": true, - "sha256": "0bdyx7dj9n6qmja3qqaznc6kfg1x8g56b91hmdv8nyhxik720cid" - }, "jdk16": { "link": "https://github.com/adoptium/temurin16-binaries/releases/download/jdk-16.0.2%2B7/OpenJDK16U-jdk_x64_linux_hotspot_16.0.2_7.tar.gz", "major_version": 16, @@ -43,6 +15,34 @@ "java_version": "1.8.0_332-b09", "early_access": false, "sha256": "1nl37nyy6bcw91ym0529mpjh7chysq88zd416h57zms00l53mhdd" + }, + "jdk18": { + "link": "https://github.com/adoptium/temurin18-binaries/releases/download/jdk-18.0.1%2B10/OpenJDK18U-jdk_x64_linux_hotspot_18.0.1_10.tar.gz", + "major_version": 18, + "java_version": "18.0.1+10", + "early_access": false, + "sha256": "18mhq6bqhbig58w2n0iw2ssz9ir48dk9rzd10jpmgh92bzbxkc8n" + }, + "jdk17": { + "link": "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.3%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.3_7.tar.gz", + "major_version": 17, + "java_version": "17.0.3+7", + "early_access": false, + "sha256": "0wvbvv01jiwkj84kqnbnqfivjy0cca8h7d8996qgpybp239bxxc1" + }, + "jdk19": { + "link": "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-2022-06-09-19-11-beta/OpenJDK-jdk_x64_linux_hotspot_2022-06-09-03-31.tar.gz", + "major_version": 19, + "java_version": "19-beta+25-202206090350", + "early_access": true, + "sha256": "0bdyx7dj9n6qmja3qqaznc6kfg1x8g56b91hmdv8nyhxik720cid" + }, + "jdk11": { + "link": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.15%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.15_10.tar.gz", + "major_version": 11, + "java_version": "11.0.15+10", + "early_access": false, + "sha256": "02sdnmk9zswi5skhyik1zaqm08v6cyggjc7c7ykwrw322rd4vnsz" } }, "latest": { @@ -68,27 +68,105 @@ } }, "semeru": { - "versions": {}, + "versions": { + "jdk16": { + "link": "https://github.com/AdoptOpenJDK/semeru16-binaries/releases/download/jdk-16.0.2%2B7_openj9-0.27.0/ibm-semeru-open-jdk_x64_linux_16.0.2_7_openj9-0.27.0.tar.gz", + "major_version": 16, + "java_version": "16.0.2+7", + "early_access": false, + "sha256": "0vl8q3v35j5229qvxz64lp25f0vk4q0shrjdk2hr3x4s3ndfnj8k" + }, + "jdk12": { + "link": "https://github.com/AdoptOpenJDK/openjdk12-binaries/releases/download/jdk-12.0.2%2B10_openj9-0.15.1/OpenJDK12U-jdk_x64_linux_openj9_12.0.2_10_openj9-0.15.1.tar.gz", + "major_version": 12, + "java_version": "12.0.2+10", + "early_access": false, + "sha256": "0ab22d1603c0j40df3qamgq3cnvfhm3w22lqc1y23pqdaqhz1xp2" + }, + "jdk10": { + "link": "https://github.com/AdoptOpenJDK/openjdk10-openj9-releases/releases/download/jdk-10.0.2%2B13_openj9-0.9.0/OpenJDK10-OPENJ9_x64_Linux_jdk-10.0.2.13_openj9-0.9.0.tar.gz", + "major_version": 10, + "java_version": "10.0.2+13", + "early_access": false, + "sha256": "1lbr2y1ld57s5kbdan254mq5kf7w960yym0q15k3cbrvhnrxmw0y" + }, + "jdk13": { + "link": "https://github.com/AdoptOpenJDK/openjdk13-binaries/releases/download/jdk-13.0.2%2B8_openj9-0.18.0/OpenJDK13U-jdk_x64_linux_openj9_13.0.2_8_openj9-0.18.0.tar.gz", + "major_version": 13, + "java_version": "13.0.2+8", + "early_access": false, + "sha256": "1415qpv9gwzrz3cxfrlkqa84ii2xgvwkqy873nw7v10c1p9zdv5f" + }, + "jdk14": { + "link": "https://github.com/AdoptOpenJDK/openjdk14-binaries/releases/download/jdk-14.0.2%2B12_openj9-0.21.0/OpenJDK14U-jdk_x64_linux_openj9_14.0.2_12_openj9-0.21.0.tar.gz", + "major_version": 14, + "java_version": "14.0.2+12", + "early_access": false, + "sha256": "1gpkrbk64dhry1jfkcl39ha8h877pnpnrhvfb7ralpdnrlw72vrh" + }, + "jdk18": { + "link": "https://github.com/AdoptOpenJDK/semeru18-binaries/releases/download/jdk-18.0.1%2B10_openj9-0.32.0/ibm-semeru-open-jdk_x64_linux_18.0.1_10_openj9-0.32.0.tar.gz", + "major_version": 18, + "java_version": "18.0.1+10", + "early_access": false, + "sha256": "1spm6ypyrwa8j85kpz4v7337kad8v94wacnfr1rk2klw2q1n253f" + }, + "jdk15": { + "link": "https://github.com/AdoptOpenJDK/openjdk15-binaries/releases/download/jdk-15.0.2%2B7_openj9-0.24.0/OpenJDK15U-jdk_x64_linux_openj9_15.0.2_7_openj9-0.24.0.tar.gz", + "major_version": 15, + "java_version": "15.0.2+7", + "early_access": false, + "sha256": "05j807m13rlfnx7xjihf210vpxvj2bzdb02vc658sphvkgbwq5am" + }, + "jdk11": { + "link": "https://github.com/AdoptOpenJDK/semeru11-binaries/releases/download/jdk-11.0.15%2B10_openj9-0.32.0/ibm-semeru-open-jdk_x64_linux_11.0.15_10_openj9-0.32.0.tar.gz", + "major_version": 11, + "java_version": "11.0.15+10", + "early_access": false, + "sha256": "135gpvhcncw5j3n7yhfhhlcnpfzq605m4v8m1l2712q77vfdfqvh" + }, + "jdk8": { + "link": "https://github.com/AdoptOpenJDK/semeru8-binaries/releases/download/jdk8u332-b09_openj9-0.32.0/ibm-semeru-open-jdk_x64_linux_8u332b09_openj9-0.32.0.tar.gz", + "major_version": 8, + "java_version": "1.8.0_332-b09", + "early_access": false, + "sha256": "1ivm8wccxa4j04vvmvymgfb23kd3jl33mhap4mlja9gkpj92vzxy" + }, + "jdk9": { + "link": "https://github.com/AdoptOpenJDK/openjdk9-openj9-releases/releases/download/jdk-9.0.4%2B12_openj9-0.9.0/OpenJDK9-OPENJ9_x64_Linux_jdk-9.0.4.12_openj9-0.9.0.tar.gz", + "major_version": 9, + "java_version": "9.0.4+12", + "early_access": false, + "sha256": "0svf3rvbpjzg85v6nxwcy6n45v5byjpihiwil2msdhagxnvl3fym" + }, + "jdk17": { + "link": "https://github.com/AdoptOpenJDK/semeru17-binaries/releases/download/jdk-17.0.3%2B7_openj9-0.32.0/ibm-semeru-open-jdk_x64_linux_17.0.3_7_openj9-0.32.0.tar.gz", + "major_version": 17, + "java_version": "17.0.3+7", + "early_access": false, + "sha256": "1pdvj0spbsnai6hk5s3ciid9mgqbcrxmc3fn4b67b5jk7m9h9jkx" + } + }, "latest": { - "link": "", - "major_version": 0, - "java_version": "", + "link": "https://github.com/AdoptOpenJDK/semeru18-binaries/releases/download/jdk-18.0.1%2B10_openj9-0.32.0/ibm-semeru-open-jdk_x64_linux_18.0.1_10_openj9-0.32.0.tar.gz", + "major_version": 18, + "java_version": "18.0.1+10", "early_access": false, - "sha256": "" + "sha256": "1spm6ypyrwa8j85kpz4v7337kad8v94wacnfr1rk2klw2q1n253f" }, "stable": { - "link": "", - "major_version": 0, - "java_version": "", + "link": "https://github.com/AdoptOpenJDK/semeru18-binaries/releases/download/jdk-18.0.1%2B10_openj9-0.32.0/ibm-semeru-open-jdk_x64_linux_18.0.1_10_openj9-0.32.0.tar.gz", + "major_version": 18, + "java_version": "18.0.1+10", "early_access": false, - "sha256": "" + "sha256": "1spm6ypyrwa8j85kpz4v7337kad8v94wacnfr1rk2klw2q1n253f" }, "lts": { - "link": "", - "major_version": 0, - "java_version": "", + "link": "https://github.com/AdoptOpenJDK/semeru17-binaries/releases/download/jdk-17.0.3%2B7_openj9-0.32.0/ibm-semeru-open-jdk_x64_linux_17.0.3_7_openj9-0.32.0.tar.gz", + "major_version": 17, + "java_version": "17.0.3+7", "early_access": false, - "sha256": "" + "sha256": "1pdvj0spbsnai6hk5s3ciid9mgqbcrxmc3fn4b67b5jk7m9h9jkx" } } } diff --git a/updater/src/adoptium.rs b/updater/src/adoptium.rs index 7d828ce..ef26421 100644 --- a/updater/src/adoptium.rs +++ b/updater/src/adoptium.rs @@ -27,7 +27,6 @@ pub struct Package { checksum_link: String, download_count: u64, pub link: String, - metadata_link: String, name: String, size: u64, } @@ -43,7 +42,6 @@ pub struct Binary { os: String, pub package: Package, project: String, - scm_ref: String, updated_at: String, } @@ -140,12 +138,13 @@ pub async fn get_available_releases(client: &Client) -> Result R os: "linux".to_string(), page_size: PAGE_SIZE, project: "jdk".to_string(), + jvm_impl: "hotspot".to_string(), }) .map_err(|e| eyre!(e)) .context("Failed to build request")? diff --git a/updater/src/main.rs b/updater/src/main.rs index 49c4db0..90220ed 100644 --- a/updater/src/main.rs +++ b/updater/src/main.rs @@ -9,6 +9,8 @@ use surf::Client; /// Adoptium API pub mod adoptium; +/// Semeru API +pub mod semeru; /// Java release struct #[derive(Deserialize, Serialize, Debug, Clone, Default)] @@ -75,7 +77,7 @@ async fn main() -> Result<()> { // Get adoptium releases let adoptium_releases = get_adoptium_releases(&client).await?; // Spit out to the serialization format - let adoptium = Sources { + let temurin = Sources { versions: adoptium_releases .clone() .into_iter() @@ -94,10 +96,30 @@ async fn main() -> Result<()> { .expect("Missing release") .clone(), }; - let system = System { - temurin: adoptium, - ..Default::default() + // Get semeru releases + + let semeru_releases = get_semeru_releases(&client).await?; + // Spit out to the serialization format + let semeru = Sources { + versions: semeru_releases + .clone() + .into_iter() + .map(|(k, v)| (format!("jdk{}", k), v)) + .collect(), + latest: semeru_releases + .get(&available.most_recent_feature_release) + .expect("Missing release") + .clone(), + stable: semeru_releases + .get(&available.most_recent_feature_release) + .expect("Missing release") + .clone(), + lts: semeru_releases + .get(<s_version) + .expect("Missing release") + .clone(), }; + let system = System { temurin, semeru }; let mut systems = HashMap::new(); systems.insert("x86_64-linux".to_string(), system); let output = serde_json::to_string_pretty(&systems).context("Failed to encode sources")?; @@ -107,7 +129,21 @@ async fn main() -> Result<()> { /// Get the releases from adoptium pub async fn get_adoptium_releases(client: &Client) -> Result> { - let releases: Result> = adoptium::get_releases(&client) + let releases: Result> = adoptium::get_releases(client) + .await? + .into_iter() + .map(|(key, val)| match val.try_into() { + Ok(val) => Ok((key, val)), + Err(err) => Err(err), + }) + .collect(); + + releases.context("Failed getting release from adoptium") +} + +/// Get the releases from semeru +pub async fn get_semeru_releases(client: &Client) -> Result> { + let releases: Result> = semeru::get_releases(client) .await? .into_iter() .map(|(key, val)| match val.try_into() { diff --git a/updater/src/semeru.rs b/updater/src/semeru.rs new file mode 100644 index 0000000..96541da --- /dev/null +++ b/updater/src/semeru.rs @@ -0,0 +1,95 @@ +use std::collections::HashMap; + +use color_eyre::{ + eyre::{eyre, Context, Result}, + Help, SectionExt, +}; +use surf::Client; + +use crate::adoptium::{AvailableReleases, Release, ReleaseQuery}; + +/// Page size +pub const PAGE_SIZE: u64 = 10; + +/// Attempts to get the available releases +pub async fn get_available_releases(client: &Client) -> Result { + let endpoint = "https://api.adoptopenjdk.net/v3/info/available_releases?jvm_impl=openj9"; + client + .get(endpoint) + .recv_json() + .await + .map_err(|e| eyre!(e)) + .context("Failed to request available versions from semeru") + .with_section(|| endpoint.to_string().header("Failed Request:")) +} + +/// Attempts to get the release info for a particular version +pub async fn get_release(client: &Client, version: u64, release_type: &str) -> Result { + let endpoint = format!( + "https://api.adoptopenjdk.net/v3/assets/feature_releases/{}/{}", + version, release_type + ); + let request = client + .get(endpoint) + .query(&ReleaseQuery { + architecture: "x64".to_string(), + heap_size: "normal".to_string(), + image_type: "jdk".to_string(), + os: "linux".to_string(), + page_size: PAGE_SIZE, + project: "jdk".to_string(), + jvm_impl: "openj9".to_string(), + }) + .map_err(|e| eyre!(e)) + .context("Failed to build request")? + .build(); + let query = request.url().as_str().to_string(); + let mut releases: Vec = client + .recv_json(request) + .await + .map_err(|e| eyre!(e)) + .context("Failed to get release information from semeru") + .with_section(move || query.header("Failed Request"))?; + releases.sort(); + match releases.pop() { + Some(release) => Ok(release), + None => Err(eyre!("Semeru endpoint did not return any valid releases")), + } +} + +/// Attempts to get all the versions +pub async fn get_releases(client: &Client) -> Result> { + let available = get_available_releases(client) + .await + .context("Failed to list semeru releases")?; + let mut output = HashMap::new(); + // Get the generally available version of all the available releases + for version in available.available_releases { + let release = get_release(client, version, "ga").await.with_context(|| { + format!("Failed to get version {} from the semeru archive", version) + })?; + output.insert(version, release); + } + // See if we already have the latest version + if output.contains_key(&available.most_recent_feature_version) { + // Go ahead and return + Ok(output) + } else { + let version = available.most_recent_feature_version; + // Otherwise try to get an EA version of it + + match get_release(client, version, "ea").await { + Ok(release) => { + output.insert(version, release); + } + Err(e) => { + eprintln!( + "Failed to get version {} (latest) from the semeru archive: {:?}", + version, e + ) + } + } + + Ok(output) + } +}