Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/bazel-maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@ name: Bazel patch compatibility (Maven)
# patch is one block in MODULE.bazel that puts a repository with the
# patched jar in the place of the hub's own (`override_repo`): the flows
# hold the block's text, the patched build and its sha256 check, the undo,
# a version bump of the patched artifact, and a hub the block cannot
# patch. bazel-compatibility.yml holds the legs and what every ecosystem
# a hosted scan that finds the artifact in the hub's lock, a version bump
# of the patched artifact, and a hub the block cannot patch.
# bazel-compatibility.yml holds the legs and what every ecosystem
# shares; this file says when the Maven legs run and on which releases.
#
# Each major release has an era of its own, because the ruleset decides
Expand Down
12 changes: 7 additions & 5 deletions crates/socket-patch-cli/CLI_CONTRACT.md

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions crates/socket-patch-cli/src/commands/get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -926,6 +926,11 @@ async fn filter_to_installed_purls(
let hosted = mode == super::scan::ScanMode::Hosted;
let supplement = super::scan::project_lockfile_supplement(&ctx, &[], None, hosted).await;
pnp_diags = supplement.unsupported;
// What was not read of a Bazel workspace's Maven hubs says nothing
// of patches that are for no Maven artifact.
let maven = accessible.iter().any(|p| p.purl.starts_with("pkg:maven/"));
let hub = socket_patch_core::bazel::maven::HUB_NOT_READ;
pnp_diags.retain(|diag| maven || diag.code != hub);
if mode != super::scan::ScanMode::Agent {
present.extend(supplement.entries.iter().map(|e| PurlKey::new(&e.purl)));
// A hosted run's supplement also names the gems the lock
Expand Down
49 changes: 44 additions & 5 deletions crates/socket-patch-cli/src/commands/scan/discovery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,14 @@ use futures_util::StreamExt;
use socket_patch_core::api::client::hold_back_debug;
use socket_patch_core::api::ranking::{batch_supersedes, cmp_batch_infos};
use socket_patch_core::api::types::{BatchPackagePatches, PatchResponse, PatchSearchResult};
use socket_patch_core::bazel::maven::HubJars;
use socket_patch_core::crawlers::Ecosystem;
use socket_patch_core::manifest::schema::{PatchManifest, PatchRecord};
use socket_patch_core::patch::redirect::upstream::HostedPin;
use socket_patch_core::utils::concurrent::{api_concurrency_for, ordered_concurrent};
use socket_patch_core::utils::purl::{normalize_purl, strip_purl_qualifiers};
use socket_patch_core::utils::purl_key::PurlKey;
use socket_patch_core::vendor::lock_inventory::LockfileEntry;
use socket_patch_core::vendor::lock_inventory::{LockfileEntry, UnsupportedNpmLayout};
use socket_patch_core::vendor::VendorState;
use std::collections::{BTreeSet, HashMap, HashSet};

Expand Down Expand Up @@ -41,7 +44,14 @@ pub(crate) struct LockfileSupplement {
/// npm layouts the lockfile inventory REFUSED (Plug'n'Play loaders).
/// Scan surfaces these as refusal warnings: under PnP the installed-tree
/// crawl is empty too, so otherwise the project scans as a silent no-op.
pub(crate) unsupported: Vec<socket_patch_core::vendor::lock_inventory::UnsupportedNpmLayout>,
/// Also what a hosted run did not read of the hubs of a Bazel workspace
/// ([`hub_jars`]).
pub(crate) unsupported: Vec<UnsupportedNpmLayout>,
/// The entries of the block in `MODULE.bazel` whose jar no hub of the
/// Bazel workspace fetches any more ([`hub_jars`]), as hosted pins of
/// that file alone: Bazel stops at each, so a hosted scan takes them
/// out (`withdrawn::sweep`).
pub(crate) stale_hub_pins: Vec<HostedPin>,
}

/// Inventory the project's lockfile(s) and fabricate crawl entries for
Expand All @@ -61,7 +71,8 @@ pub(crate) struct LockfileSupplement {
/// (1.18.10-arm64-darwin)`: no one `.gem` stands for it, so it cannot be
/// vendored from the lock, and an in-place patch needs the installed copy).
/// Hosted mode redirects such a gem to its per-platform prebuilds from the
/// lock alone, so a hosted supplement names it too.
/// lock alone, so a hosted supplement names it too. It also names the jars
/// of [`hub_jars`], and what was not read of the hubs joins `unsupported`.
pub(crate) async fn lockfile_supplement(
ctx: &crate::commands::context::ProjectContext<'_>,
crawled: &[socket_patch_core::crawlers::types::CrawledPackage],
Expand All @@ -81,18 +92,46 @@ pub(crate) async fn lockfile_supplement(
} else {
Vec::new()
};
if entries.is_empty() && platform_gems.is_empty() {
let HubJars {
jars,
unread,
stale,
} = hub_jars(common, hosted).await;
out.unsupported
.extend(unread.map(|(code, detail)| UnsupportedNpmLayout { code, detail }));
let files = vec![socket_patch_core::bazel::maven::FILE.to_string()];
let pin = |(purl, uuid)| HostedPin {
purl,
uuid,
files: files.clone(),
};
out.stale_hub_pins = stale.into_iter().map(pin).collect();
if entries.is_empty() && platform_gems.is_empty() && jars.is_empty() {
return out;
}
let resolved = entries
.iter()
.map(|entry| entry.purl.as_str())
.chain(platform_gems.iter().map(String::as_str));
.chain(platform_gems.iter().map(String::as_str))
.chain(jars.iter().map(String::as_str));
(out.packages, out.purls) = lockfile_only_packages(resolved, crawled, only, &common.cwd);
out.entries = entries.clone();
out
}

/// The purls of the jars that the pinned rules_jvm_external hubs of a Bazel
/// workspace fetch when `--cwd` is its root, what was not read of its hubs
/// and the entries of its block that are stale
/// ([`socket_patch_core::bazel::maven::hub_jars`]). Only for a hosted run
/// with Maven in scope: no other mode patches a hub. The callers are runs
/// of a project, never of the global installs.
pub(crate) async fn hub_jars(common: &GlobalArgs, hosted: bool) -> HubJars {
if !hosted || !common.ecosystem_selected(Ecosystem::Maven) {
return HubJars::default();
}
socket_patch_core::bazel::maven::hub_jars(&common.cwd).await
}

/// The packages `resolved` (the purls the project's lockfiles resolve) with
/// no crawled counterpart, fabricated as crawl entries, plus their
/// [`PurlKey`]s. "Crawled" is by [`PurlKey`], the same relation
Expand Down
121 changes: 87 additions & 34 deletions crates/socket-patch-cli/src/commands/scan/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1605,6 +1605,38 @@ fn emit_scan_error(common: &GlobalArgs, error: EnvelopeError) {
emit_scan(&env);
}

/// A restore of `sweep` that failed part-way and could not put its files
/// back leaves the lockfiles half restored: the run stops there with exit
/// 1 (`Err`), rather than write over them. The restore printed the
/// failure as it happened; `counts` are the scanned and the lockfile-only
/// packages of the envelope, and the last argument is what the failed
/// scan's telemetry takes. Otherwise `sweep` goes on.
async fn stop_half_restored(
common: &GlobalArgs,
sweep: Option<withdrawn::WithdrawnSweep>,
counts: (usize, usize),
paths: &crate::path_scope::PathScope,
(fallback_to_proxy, mode, auth): (bool, &str, &TelemetryAuth),
) -> Result<Option<withdrawn::WithdrawnSweep>, i32> {
let half = |sweep: &withdrawn::WithdrawnSweep| sweep.half_restored().map(str::to_string);
let Some(why) = sweep.as_ref().and_then(half) else {
return Ok(sweep);
};
track_patch_scan_failed("lockfile_write_failed", &why, fallback_to_proxy, mode, auth).await;
if common.json {
let mut env = scan_envelope(common);
env.set_extra("scannedPackages", serde_json::json!(counts.0));
env.set_extra("lockfileOnlyPackages", serde_json::json!(counts.1));
env.set_extra("paths", serde_json::json!(paths.raw()));
if let Some(sweep) = sweep {
sweep.record_into(&mut env);
}
env.mark_error(EnvelopeError::new("lockfile_write_failed", why));
emit_scan(&env);
}
Err(1)
}

/// `(purl, n)` pairs summed per ecosystem, by CLI name: how many packages
/// (or patches) of each ecosystem a scan found. An ecosystem with nothing
/// has no entry; a purl of no known ecosystem is not counted.
Expand Down Expand Up @@ -2267,11 +2299,23 @@ async fn scan_project(
if args.common.json || args.vex.vex.is_some() {
unscanned_projects = Some(others);
} else {
// The root of a Bazel workspace can be without a file
// that makes `--cwd` a project, and a hosted scan has work
// there all the same: the jars that its pinned
// rules_jvm_external hubs fetch, a hub to name as not
// read, or a stale entry of the block to take out.
let mut dirs = found.dirs;
if !dirs.iter().any(|(_, explicit)| *explicit) {
let hosted = args.mode == Some(ScanMode::Hosted);
if discovery::hub_jars(&args.common, hosted).await != Default::default() {
dirs.insert(0, (args.common.cwd.clone(), true));
}
}
return Box::pin(run_project_dirs(
args,
invocation,
mode_inferred,
found.dirs,
dirs,
report,
))
.await;
Expand Down Expand Up @@ -2912,6 +2956,14 @@ async fn scan_project(

let all_purls: Vec<String> = filtered_crawled.iter().map(|p| p.purl.clone()).collect();
let package_count = all_purls.len();
// The stale entries of a Bazel workspace's block that this run may
// take out: the policy holds them as it holds the run's packages.
let stale_hub_pins: Vec<_> = lockfile_only
.stale_hub_pins
.iter()
.filter(|pin| policy.admits_restore(&pin.purl))
.cloned()
.collect();

if package_count == 0 {
// An empty crawl is where the projects below matter most: they
Expand Down Expand Up @@ -2943,6 +2995,21 @@ async fn scan_project(
);
}
status.finish();
// Bazel stops at an entry of the block whose jar no hub fetches
// any more, whatever the project holds: such an entry is taken
// out here too (`withdrawn::sweep`, which asks nothing for it).
let stale_sweep = if hosted && policy.writes_allowed() {
Some(withdrawn::sweep(&args.common, &api_client, &[], &stale_hub_pins).await)
} else {
None
};
let telemetry = (false, requested_mode, &telemetry_auth);
let stopped =
stop_half_restored(&args.common, stale_sweep, (0, 0), &path_scope, telemetry).await;
let stale_sweep = match stopped {
Ok(sweep) => sweep,
Err(code) => return code,
};
if human {
print_layout_refusals(&layout_refusals, &project_patch_notice, args.common.silent);
policy.print_warnings(args.common.silent);
Expand Down Expand Up @@ -3009,6 +3076,9 @@ async fn scan_project(
if prune {
env.warnings.push(hosted::prune_ignored_warning());
}
if let Some(sweep) = stale_sweep {
sweep.record_into(&mut env);
}
env.set_extra("redirect", hosted::redirect_block(Vec::new()));
} else if !vendor {
// `redirectState` rides the empty-discovery envelope too
Expand Down Expand Up @@ -3388,43 +3458,26 @@ async fn scan_project(
// longer serves, before the rewrite below re-pins any replacement.
// Runs ahead of every "nothing found" exit: a package whose only patch
// was withdrawn has no search results at all.
let mut withdrawn_sweep = if hosted && !args.common.is_global() && policy.writes_allowed() {
let withdrawn_sweep = if hosted && !args.common.is_global() && policy.writes_allowed() {
let pins = withdrawn::pins_in_scope(&hosted_pin_list, &all_purls);
Some(withdrawn::sweep(&args.common, &api_client, &pins).await)
Some(withdrawn::sweep(&args.common, &api_client, &pins, &stale_hub_pins).await)
} else {
None
};
// A restore that failed part-way and could not put its files back
// leaves the lockfiles half restored: stop, rather than rewrite over
// them. (The restore printed the failure as it happened.)
if let Some(why) = withdrawn_sweep
.as_ref()
.and_then(|sweep| sweep.half_restored().map(str::to_string))
{
track_patch_scan_failed(
"lockfile_write_failed",
&why,
fallback_to_proxy,
requested_mode,
&telemetry_auth,
)
.await;
if args.common.json {
let mut env = scan_envelope(&args.common);
env.set_extra("scannedPackages", serde_json::json!(package_count));
env.set_extra(
"lockfileOnlyPackages",
serde_json::json!(lockfile_only_count),
);
env.set_extra("paths", serde_json::json!(path_scope.raw()));
if let Some(sweep) = withdrawn_sweep {
sweep.record_into(&mut env);
}
env.mark_error(EnvelopeError::new("lockfile_write_failed", why));
emit_scan(&env);
}
return 1;
}
let counts = (package_count, lockfile_only_count);
let telemetry = (fallback_to_proxy, requested_mode, &telemetry_auth);
let stopped = stop_half_restored(
&args.common,
withdrawn_sweep,
counts,
&path_scope,
telemetry,
)
.await;
let mut withdrawn_sweep = match stopped {
Ok(sweep) => sweep,
Err(code) => return code,
};
// The sweep's reference answers, reused by the hosted redirect.
let sweep_references = withdrawn_sweep
.as_mut()
Expand Down
8 changes: 8 additions & 0 deletions crates/socket-patch-cli/src/commands/scan/policy.rs
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,14 @@ impl ScanPolicy {
self.policy.enabled()
}

/// Whether the run may take out a pin of `purl` that none of its
/// packages stands for (a stale entry of a Bazel workspace's block):
/// not in a root the policy filtered, and not for a package it
/// filters, as for a pin of a package the run covers.
pub(crate) fn admits_restore(&self, purl: &str) -> bool {
self.root_verdict.is_ok() && self.policy.admits_purl(purl).is_ok()
}

/// Set the merged recorded view (manifest > hosted pins > vendor ledger).
pub(crate) fn set_recorded(&mut self, merged: Option<&PatchManifest>) {
self.recorded = merged
Expand Down
Loading
Loading