Fix checksum test by reducing network calls (#444)

This test was originally starting with an empty set of checksums,
leading to the download of a checksum for every released and snapshot
version. This resulted in in sporadic test failures.

We now start with a known set of checksums and ensure that those that
are missing are downloaded. This involved some refactoring and
improvement in the way snapshot checksums are processed.

sources/src/wrapper-validation/checksums.ts

@@ -38,7 +38,7 @@ export const KNOWN_CHECKSUMS = loadKnownChecksums()
 export async function fetchUnknownChecksums(
     allowSnapshots: boolean,
     knownChecksums: WrapperChecksums
-): Promise<Set<string>> {
+): Promise<WrapperChecksums> {
     const all = await httpGetJsonArray('https://services.gradle.org/versions/all')
     const withChecksum = all.filter(
         entry => typeof entry === 'object' && entry != null && entry.hasOwnProperty('wrapperChecksumUrl')
@@ -51,20 +51,21 @@ export async function fetchUnknownChecksums(
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         (entry: any) => !knownChecksums.versions.has(entry.version)
     )
-    const checksumUrls = notKnown.map(
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        (entry: any) => entry.wrapperChecksumUrl as string
-    )
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const checksumUrls = notKnown.map((entry: any) => [entry.version, entry.wrapperChecksumUrl] as [string, string])
     if (allowSnapshots) {
-        await addDistributionSnapshotChecksums(checksumUrls)
+        await addDistributionSnapshotChecksumUrls(checksumUrls)
     }
-    const checksums = await Promise.all(
-        checksumUrls.map(async (url: string) => {
-            // console.log(`Fetching checksum from ${url}`)
-            return httpGetText(url)
+
+    const wrapperChecksums = new WrapperChecksums()
+    await Promise.all(
+        checksumUrls.map(async ([version, url]) => {
+            const checksum = await httpGetText(url)
+            wrapperChecksums.add(version, checksum)
         })
     )
-    return new Set(checksums)
+    return wrapperChecksums
 }
 
 async function httpGetJsonArray(url: string): Promise<unknown[]> {
@@ -76,21 +77,20 @@ async function httpGetText(url: string): Promise<string> {
     return await response.readBody()
 }
 
-// Public for testing
-export async function addDistributionSnapshotChecksums(checksumUrls: string[]): Promise<void> {
-    // Load the index page of the distribution snapshot repository
+async function addDistributionSnapshotChecksumUrls(checksumUrls: [string, string][]): Promise<void> {
+    // Load the index page of the distribution snapshot repository into cheerio
     const indexPage = await httpGetText('https://services.gradle.org/distributions-snapshots/')
-
-    // // Extract all wrapper checksum from the index page. These end in -wrapper.jar.sha256
-    // // Load the HTML into cheerio
     const $ = cheerio.load(indexPage)
 
     // // Find all links ending with '-wrapper.jar.sha256'
     const wrapperChecksumLinks = $('a[href$="-wrapper.jar.sha256"]')
-
-    // build the absolute URL for each wrapper checksum
     wrapperChecksumLinks.each((index, element) => {
-        const url = $(element).attr('href')
-        checksumUrls.push(`https://services.gradle.org${url}`)
+        const url = $(element).attr('href')!
+
+        // Extract the version from the url
+        const version = url.match(/\/distributions-snapshots\/gradle-(.*?)-wrapper\.jar\.sha256/)?.[1]
+        if (version) {
+            checksumUrls.push([version, `https://services.gradle.org${url}`])
+        }
     })
 }

sources/src/wrapper-validation/validate.ts

@@ -33,7 +33,7 @@ export async function findInvalidWrapperJars(
             const fetchedValidChecksums = await checksums.fetchUnknownChecksums(allowSnapshots, knownValidChecksums)
 
             for (const wrapperJar of notYetValidatedWrappers) {
-                if (!fetchedValidChecksums.has(wrapperJar.checksum)) {
+                if (!fetchedValidChecksums.checksums.has(wrapperJar.checksum)) {
                     result.invalid.push(wrapperJar)
                 } else {
                     result.valid.push(wrapperJar)