rewrite updater

This commit is contained in:
Nathan McCarty 2022-11-29 02:21:44 -05:00
parent 83f8b7c051
commit a7ca755775
Signed by: thatonelutenist
GPG Key ID: D70DA3DD4D1E9F96
7 changed files with 713 additions and 1584 deletions

1526
updater/Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -6,8 +6,9 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
async-std = { version = "1.10.0", features = ["std", "async-global-executor", "futures-lite", "num_cpus", "attributes"], default-features = false } async-std = { version = "1.12.0", features = ["attributes"] }
color-eyre = "0.5.11" color-eyre = "0.5.11"
isahc = "1.7.2"
serde = { version = "1.0.132", features = ["derive"] } serde = { version = "1.0.132", features = ["derive"] }
serde_json = "1.0.73" serde_json = "1.0.73"
surf = { version = "2.3.2", features = ["h1-client-rustls", "encoding"], default-features = false } urlencoding = "2.1.2"

View File

@ -67,8 +67,6 @@
}; };
devBase = with pkgs; [ devBase = with pkgs; [
# Build tools # Build tools
openssl
pkg-config
rust-analyzer rust-analyzer
cmake cmake
gnuplot gnuplot
@ -92,10 +90,7 @@
# Sourcehut # Sourcehut
hut hut
]; ];
sharedDeps = with pkgs; sharedDeps = with pkgs; [ curl openssl pkg-config ];
[
];
sharedNativeDeps = with pkgs; sharedNativeDeps = with pkgs;
[ [

View File

@ -1,216 +0,0 @@
use std::collections::BTreeMap;
use color_eyre::{
eyre::{eyre, Context, Result},
Help, SectionExt,
};
use serde::{Deserialize, Serialize};
use surf::Client;
/// Page size
pub const PAGE_SIZE: u64 = 10;
/// Response from `/v3/info/available_releases` endpoint
#[derive(Deserialize, Serialize, Debug)]
pub struct AvailableReleases {
pub available_lts_releases: Vec<u64>,
pub available_releases: Vec<u64>,
pub most_recent_feature_release: u64,
pub most_recent_feature_version: u64,
pub tip_version: u64,
}
/// Package for a particular binary
#[derive(Deserialize, Serialize, Debug)]
pub struct Package {
checksum: String,
checksum_link: String,
download_count: u64,
pub link: String,
name: String,
size: u64,
}
/// Information about a particular binary
#[derive(Deserialize, Serialize, Debug)]
pub struct Binary {
architecture: String,
download_count: u64,
heap_size: String,
image_type: String,
jvm_impl: String,
os: String,
pub package: Package,
project: String,
updated_at: String,
}
/// Information about a source
#[derive(Deserialize, Serialize, Debug)]
pub struct Source {
link: String,
name: String,
size: u64,
}
/// Version data
#[derive(Deserialize, Serialize, Debug, PartialEq, Eq)]
pub struct VersionData {
pub build: u64,
pub major: u64,
pub minor: u64,
pub openjdk_version: String,
pub security: u64,
pub semver: String,
}
impl PartialOrd for VersionData {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
match self.major.partial_cmp(&other.major) {
Some(std::cmp::Ordering::Equal) => {}
ord => return ord,
}
match self.minor.partial_cmp(&other.minor) {
Some(std::cmp::Ordering::Equal) => {}
ord => return ord,
}
match self.security.partial_cmp(&other.security) {
Some(std::cmp::Ordering::Equal) => {}
ord => return ord,
}
self.build.partial_cmp(&other.build)
}
}
impl Ord for VersionData {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.partial_cmp(other).unwrap()
}
}
/// Information about a particular feature release
#[derive(Deserialize, Serialize, Debug)]
pub struct Release {
pub binaries: Vec<Binary>,
download_count: u64,
id: String,
release_link: String,
pub release_type: String,
source: Option<Source>,
timestamp: String,
updated_at: String,
vendor: String,
pub version_data: VersionData,
}
impl PartialEq for Release {
fn eq(&self, other: &Self) -> bool {
self.version_data == other.version_data
}
}
impl Eq for Release {}
impl PartialOrd for Release {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.version_data.partial_cmp(&other.version_data)
}
}
impl Ord for Release {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.version_data.cmp(&other.version_data)
}
}
/// Attempts to get the available releases
pub async fn get_available_releases(client: &Client) -> Result<AvailableReleases> {
let endpoint = "https://api.adoptium.net/v3/info/available_releases";
client
.get(endpoint)
.recv_json()
.await
.map_err(|e| eyre!(e))
.context("Failed to request available versions from adoptium")
.with_section(|| endpoint.to_string().header("Failed Request:"))
}
/// Release query struct
#[derive(Deserialize, Serialize, Debug)]
pub struct ReleaseQuery {
pub architecture: String,
pub heap_size: String,
pub image_type: String,
pub jvm_impl: String,
pub os: String,
pub page_size: u64,
pub project: String,
}
/// Attempts to get the release info for a particular version
pub async fn get_release(client: &Client, version: u64, release_type: &str) -> Result<Release> {
let endpoint = format!(
"https://api.adoptium.net/v3/assets/feature_releases/{}/{}",
version, release_type
);
let request = client
.get(endpoint)
.query(&ReleaseQuery {
architecture: "x64".to_string(),
heap_size: "normal".to_string(),
image_type: "jdk".to_string(),
os: "linux".to_string(),
page_size: PAGE_SIZE,
project: "jdk".to_string(),
jvm_impl: "hotspot".to_string(),
})
.map_err(|e| eyre!(e))
.context("Failed to build request")?
.build();
let query = request.url().as_str().to_string();
let mut releases: Vec<Release> = client
.recv_json(request)
.await
.map_err(|e| eyre!(e))
.context("Failed to get release information from adoptium")
.with_section(move || query.header("Failed Request"))?;
releases.sort();
match releases.pop() {
Some(release) => Ok(release),
None => Err(eyre!("Adoptium endpoint did not return any valid releases")),
}
}
/// Attempts to get all the versions
pub async fn get_releases(client: &Client) -> Result<BTreeMap<u64, Release>> {
let available = get_available_releases(client)
.await
.context("Failed to list adoptium releases")?;
let mut output = BTreeMap::new();
// Get the generally available version of all the available releases
for version in available.available_releases {
let release = get_release(client, version, "ga").await.with_context(|| {
format!(
"Failed to get version {} from the adoptium archive",
version
)
})?;
output.insert(version, release);
}
// See if we already have the latest version
if output.contains_key(&available.most_recent_feature_version) {
// Go ahead and return
Ok(output)
} else {
let version = available.most_recent_feature_version;
// Otherwise try to get an EA version of it
let release = get_release(client, version, "ea").await.with_context(|| {
format!(
"Failed to get version {} (latest) from the adoptium archive",
version
)
})?;
output.insert(version, release);
Ok(output)
}
}

244
updater/src/api.rs Normal file
View File

@ -0,0 +1,244 @@
use std::collections::{BTreeMap};
use async_std::io::ReadExt;
use color_eyre::{
eyre::{Context, ContextCompat},
Result,
};
use isahc::HttpClient;
use serde::{Deserialize, Serialize};
/// Abstraction over an adoptium API instance
pub struct AdoptiumAPI {
/// Base URL
base_url: String,
/// Client
client: HttpClient,
/// Jvm_Impl string
jvm_impl: String,
}
impl AdoptiumAPI {
/// Create a new adoptium api pointing at adoptium
pub fn adoptium() -> Result<Self> {
Ok(AdoptiumAPI {
base_url: "https://api.adoptium.net".to_string(),
client: HttpClient::new().context("failed to open http client")?,
jvm_impl: "hotspot".to_string(),
})
}
/// Create a new api pointing at semeru
pub fn semeru() -> Result<Self> {
Ok(AdoptiumAPI {
base_url: "https://api.adoptopenjdk.net".to_string(),
client: HttpClient::new().context("failed to open http client")?,
jvm_impl: "openj9".to_string(),
})
}
/// Get the availble releases
pub async fn available_releases(&self) -> Result<AvailableReleases> {
let response = self
.client
.get_async(format!("{}/v3/info/available_releases", self.base_url))
.await
.context("Failed to request releases")?;
let mut body = response.into_body();
let mut contents = String::new();
body.read_to_string(&mut contents)
.await
.context("Failed to convert body to string")?;
let output = serde_json::from_str(&contents).context("Failed to parse json")?;
Ok(output)
}
/// Return the most recent version for the specified release and architecture
pub async fn release(
&self,
version: u32,
arch: impl AsRef<str>,
pre_release: bool,
) -> Result<Version> {
let release_type = if pre_release { "ea" } else { "ga" };
let arch = arch.as_ref();
let version = format!("[{version}, {})", version + 1);
let version = urlencoding::encode(&version);
let url = format!(
"{}/v3/info/release_versions?architecture={arch}&heap_size=normal&image_type=jdk&os_type=linux&project=jdk&release_type={release_type}&sort_method=DATE&sort_order=DESC&jvm_impl={}&version={version}",
self.base_url,
self.jvm_impl,
);
let mut response = self
.client
.get_async(url)
.await
.context("Failed to request release")?
.into_body();
let mut body = String::new();
response
.read_to_string(&mut body)
.await
.context("Failed to convert response to string")?;
let output: Versions = serde_json::from_str(&body).context("Failed to parse json")?;
let output = output.versions.get(0).context("No Versions")?;
Ok(output.clone())
}
/// Return latest release
pub async fn latest(
&self,
version: u32,
arch: impl AsRef<str>,
pre_release: bool,
) -> Result<Release> {
let release_type = if pre_release { "ea" } else { "ga" };
let arch = arch.as_ref();
let url = format!(
"{}/v3/assets/feature_releases/{version}/{release_type}?architecture={arch}&heap_size=normal&image_type=jdk&jvm_impl=hotspot&os=linux&page=0&page_size=10&project=jdk&sort_method=DATE&sort_order=DESC&jvm_impl={}",
self.base_url,
self.jvm_impl
);
let mut response = self
.client
.get_async(url)
.await
.context("Failed to request release")?
.into_body();
let mut body = String::new();
response
.read_to_string(&mut body)
.await
.context("Failed to convert response to string")?;
let releases: Vec<Release> = serde_json::from_str(&body).context("Failed to parse json")?;
let release = releases.get(0).context("No releases")?;
Ok(release.clone())
}
/// Get all versions
pub async fn get_all(&self, arch: impl AsRef<str>) -> Result<OutputReleases> {
let input_versions = self
.available_releases()
.await
.context("Failed to get releases")?;
let arch = arch.as_ref();
let mut versions: BTreeMap<String, OutputRelease> = BTreeMap::new();
for release in &input_versions.available_releases {
let output_release: OutputRelease = self
.latest(*release, arch, false)
.await
.context("Failed to get version")?
.into();
versions.insert(
format!("jdk{}", output_release.major_version),
output_release,
);
}
let latest: OutputRelease = self
.latest(input_versions.most_recent_feature_version, arch, true)
.await
.context("Failed to get version")?
.into();
let stable: OutputRelease = self
.latest(
input_versions.available_releases[input_versions.available_releases.len() - 1],
arch,
true,
)
.await
.context("Failed to get version")?
.into();
let lts: OutputRelease = self
.latest(
input_versions.available_lts_releases
[input_versions.available_lts_releases.len() - 1],
arch,
true,
)
.await
.context("Failed to get version")?
.into();
Ok(OutputReleases {
versions,
latest,
stable,
lts,
})
}
}
/// Output for an arch
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct OutputReleases {
versions: BTreeMap<String, OutputRelease>,
latest: OutputRelease,
stable: OutputRelease,
lts: OutputRelease,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct OutputRelease {
link: String,
major_version: u32,
sha256: String,
}
impl From<Release> for OutputRelease {
fn from(release: Release) -> Self {
OutputRelease {
link: release.binaries[0].package.link.clone(),
major_version: release.version_data.major,
sha256: release.binaries[0].package.checksum.clone(),
}
}
}
/// `/v3/info/available_releases`
#[derive(Debug, Serialize, Deserialize)]
pub struct AvailableReleases {
pub available_releases: Vec<u32>,
pub available_lts_releases: Vec<u32>,
pub most_recent_lts: u32,
pub most_recent_feature_release: u32,
pub most_recent_feature_version: u32,
pub tip_version: u32,
}
/// `/v3/info/release_verions`
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Version {
major: u32,
minor: u32,
security: u32,
patch: Option<u32>,
pre: Option<String>,
adopt_build_number: Option<u64>,
semver: String,
openjdk_version: String,
build: Option<u64>,
optional: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Versions {
versions: Vec<Version>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Release {
version_data: Version,
binaries: Vec<Binary>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Binary {
architecture: String,
package: Package,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Package {
checksum: String,
link: String,
name: String,
}

View File

@ -1,168 +1,60 @@
use std::{collections::BTreeMap, process::Command}; use std::collections::BTreeMap;
use api::OutputReleases;
use color_eyre::{ use color_eyre::{
eyre::{eyre, Context, Result}, eyre::{Context, Result},
Section, SectionExt,
}; };
use serde::{Deserialize, Serialize};
use surf::Client;
/// Adoptium API
pub mod adoptium;
/// Semeru API
pub mod semeru;
/// Java release struct use crate::api::AdoptiumAPI;
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct Release {
link: String,
major_version: u64,
java_version: String,
early_access: bool,
sha256: String,
}
/// Sources serialization struct /// API Abstraction
#[derive(Deserialize, Serialize, Debug, Clone, Default)] pub mod api;
pub struct Sources {
versions: BTreeMap<String, Release>,
latest: Release,
stable: Release,
lts: Release,
}
/// System serialization struct
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct System {
temurin: Sources,
semeru: Sources,
}
impl TryFrom<adoptium::Release> for Release {
type Error = color_eyre::eyre::Report;
fn try_from(value: adoptium::Release) -> Result<Self> {
if value.binaries.len() == 1 {
let package = &value.binaries[0].package;
Ok(Release {
link: package.link.clone(),
major_version: value.version_data.major,
java_version: value.version_data.openjdk_version,
early_access: value.release_type == "ea",
sha256: get_sha256(&package.link).context("Failed to prefetch package")?,
})
} else {
Err(eyre!(
"Adoptium release had an incorrect number of binaries"
))
}
}
}
#[async_std::main] #[async_std::main]
async fn main() -> Result<()> { async fn main() -> Result<()> {
color_eyre::install()?; color_eyre::install()?;
// Create a client let mut output: BTreeMap<String, BTreeMap<String, OutputReleases>> = BTreeMap::new();
let client = Client::new(); // Create the api instances
// Get list of releases from adoptium, we'll use this for some other things let adoptium = AdoptiumAPI::adoptium().context("Creating api")?;
let available = adoptium::get_available_releases(&client) let semeru = AdoptiumAPI::semeru().context("Creating api")?;
.await // Fill in x86_64 first
.context("Failed to get list of available releases")?; {
let lts_version = available let x86_64 = output.entry("x86_64-linux".to_string()).or_default();
.available_lts_releases x86_64.insert(
.iter() "temurin".to_string(),
.copied() adoptium
.max() .get_all("x64")
.expect("No LTSs?"); .await
// Get adoptium releases .context("Failed getting x86_64 adopt releases")?,
let adoptium_releases = get_adoptium_releases(&client).await?; );
// Spit out to the serialization format x86_64.insert(
let temurin = Sources { "semeru".to_string(),
versions: adoptium_releases semeru
.clone() .get_all("x64")
.into_iter() .await
.map(|(k, v)| (format!("jdk{}", k), v)) .context("Failed getting x86_64 adopt releases")?,
.collect(), );
latest: adoptium_releases }
.get(&available.most_recent_feature_version) // Then aarch64
.expect("Missing release") {
.clone(), let aarch64 = output.entry("aarch64-linux".to_string()).or_default();
stable: adoptium_releases aarch64.insert(
.get(&available.most_recent_feature_release) "temurin".to_string(),
.expect("Missing release") adoptium
.clone(), .get_all("x64")
lts: adoptium_releases .await
.get(&lts_version) .context("Failed getting aarch64 adopt releases")?,
.expect("Missing release") );
.clone(), aarch64.insert(
}; "semeru".to_string(),
// Get semeru releases semeru
.get_all("x64")
let semeru_releases = get_semeru_releases(&client).await?; .await
// Spit out to the serialization format .context("Failed getting aarch64 adopt releases")?,
let semeru = Sources { );
versions: semeru_releases }
.clone() let output = serde_json::to_string_pretty(&output).context("Failed to serialize output")?;
.into_iter() println!("{output}");
.map(|(k, v)| (format!("jdk{}", k), v))
.collect(),
latest: semeru_releases
.get(&available.most_recent_feature_release)
.expect("Missing release")
.clone(),
stable: semeru_releases
.get(&available.most_recent_feature_release)
.expect("Missing release")
.clone(),
lts: semeru_releases
.get(&lts_version)
.expect("Missing release")
.clone(),
};
let system = System { temurin, semeru };
let mut systems = BTreeMap::new();
systems.insert("x86_64-linux".to_string(), system);
let output = serde_json::to_string_pretty(&systems).context("Failed to encode sources")?;
println!("{}", output);
Ok(()) Ok(())
} }
/// Get the releases from adoptium
pub async fn get_adoptium_releases(client: &Client) -> Result<BTreeMap<u64, Release>> {
let releases: Result<BTreeMap<u64, Release>> = adoptium::get_releases(client)
.await?
.into_iter()
.map(|(key, val)| match val.try_into() {
Ok(val) => Ok((key, val)),
Err(err) => Err(err),
})
.collect();
releases.context("Failed getting release from adoptium")
}
/// Get the releases from semeru
pub async fn get_semeru_releases(client: &Client) -> Result<BTreeMap<u64, Release>> {
let releases: Result<BTreeMap<u64, Release>> = semeru::get_releases(client)
.await?
.into_iter()
.map(|(key, val)| match val.try_into() {
Ok(val) => Ok((key, val)),
Err(err) => Err(err),
})
.collect();
releases.context("Failed getting release from adoptium")
}
/// Gets the nix sha256 for a url
fn get_sha256(url: &str) -> Result<String> {
let output = Command::new("nix-prefetch-url")
.args([url, "--type", "sha256"])
.output()
.with_section(|| format!("Failed to prefetch url: {}", url).header("Prefetch Failure"))
.context("Failed to prefetch")?;
let output = String::from_utf8(output.stdout).context("Invalid utf-8 from nix pre fetch")?;
// Trim the trailing new line
Ok(output.trim().to_string())
}

View File

@ -1,95 +0,0 @@
use std::collections::BTreeMap;
use color_eyre::{
eyre::{eyre, Context, Result},
Help, SectionExt,
};
use surf::Client;
use crate::adoptium::{AvailableReleases, Release, ReleaseQuery};
/// Page size
pub const PAGE_SIZE: u64 = 10;
/// Attempts to get the available releases
pub async fn get_available_releases(client: &Client) -> Result<AvailableReleases> {
let endpoint = "https://api.adoptopenjdk.net/v3/info/available_releases?jvm_impl=openj9";
client
.get(endpoint)
.recv_json()
.await
.map_err(|e| eyre!(e))
.context("Failed to request available versions from semeru")
.with_section(|| endpoint.to_string().header("Failed Request:"))
}
/// Attempts to get the release info for a particular version
pub async fn get_release(client: &Client, version: u64, release_type: &str) -> Result<Release> {
let endpoint = format!(
"https://api.adoptopenjdk.net/v3/assets/feature_releases/{}/{}",
version, release_type
);
let request = client
.get(endpoint)
.query(&ReleaseQuery {
architecture: "x64".to_string(),
heap_size: "normal".to_string(),
image_type: "jdk".to_string(),
os: "linux".to_string(),
page_size: PAGE_SIZE,
project: "jdk".to_string(),
jvm_impl: "openj9".to_string(),
})
.map_err(|e| eyre!(e))
.context("Failed to build request")?
.build();
let query = request.url().as_str().to_string();
let mut releases: Vec<Release> = client
.recv_json(request)
.await
.map_err(|e| eyre!(e))
.context("Failed to get release information from semeru")
.with_section(move || query.header("Failed Request"))?;
releases.sort();
match releases.pop() {
Some(release) => Ok(release),
None => Err(eyre!("Semeru endpoint did not return any valid releases")),
}
}
/// Attempts to get all the versions
pub async fn get_releases(client: &Client) -> Result<BTreeMap<u64, Release>> {
let available = get_available_releases(client)
.await
.context("Failed to list semeru releases")?;
let mut output = BTreeMap::new();
// Get the generally available version of all the available releases
for version in available.available_releases {
let release = get_release(client, version, "ga").await.with_context(|| {
format!("Failed to get version {} from the semeru archive", version)
})?;
output.insert(version, release);
}
// See if we already have the latest version
if output.contains_key(&available.most_recent_feature_version) {
// Go ahead and return
Ok(output)
} else {
let version = available.most_recent_feature_version;
// Otherwise try to get an EA version of it
match get_release(client, version, "ea").await {
Ok(release) => {
output.insert(version, release);
}
Err(e) => {
eprintln!(
"Failed to get version {} (latest) from the semeru archive: {:?}",
version, e
)
}
}
Ok(output)
}
}