fix release v2 recovery flows
This commit is contained in:
1 parent
ca856f9f54
commit
4905ea6738
44 files changed
+871
-201
No files matched your search
@@ -71,15 +71,16 @@ export const getDockerReconciliationAsync = async (db: Database) => {
|
||||
const app = linkedApp ?? exactApp ?? nameApp;
|
||||
const exactUrl = Boolean(exactIntegration || exactApp);
|
||||
const linked = Boolean(integration?.appId && linkedApp);
|
||||
const integrationMoved = Boolean(integration && !exactIntegration);
|
||||
const integrationAmbiguous =
|
||||
exactIntegrationMatches.length > 1 || (exactIntegrationMatches.length === 0 && nameIntegrationMatches.length > 1);
|
||||
const appAmbiguous =
|
||||
!linkedApp && (exactAppMatches.length > 1 || (exactAppMatches.length === 0 && nameAppMatches.length > 1));
|
||||
const ambiguous = integrationAmbiguous || appAmbiguous;
|
||||
const state: DockerReconciliationState = linked
|
||||
? "linked"
|
||||
: integration && !exactUrl
|
||||
? "moved"
|
||||
const state: DockerReconciliationState = integrationMoved
|
||||
? "moved"
|
||||
: linked
|
||||
? "linked"
|
||||
: integration || app
|
||||
? "represented"
|
||||
: match
|
||||
@@ -115,6 +116,8 @@ export const getDockerReconciliationAsync = async (db: Database) => {
|
||||
app: app ? { id: app.id, name: app.name, href: app.href } : null,
|
||||
signals: {
|
||||
exactUrl,
|
||||
exactIntegrationUrl: Boolean(exactIntegration),
|
||||
exactAppUrl: Boolean(exactApp),
|
||||
nameMatch: Boolean(nameIntegration || nameApp),
|
||||
linked,
|
||||
ambiguous,
|
||||
|
||||
@@ -90,21 +90,14 @@ export const integrationRouter = createTRPCRouter({
|
||||
});
|
||||
return integrations
|
||||
.map((integration) => {
|
||||
const permissions = integration.userPermissions
|
||||
.map(({ permission }) => permission)
|
||||
.concat(integration.groupPermissions.map(({ permission }) => permission));
|
||||
const permissions = constructIntegrationPermissions(integration, ctx.session);
|
||||
|
||||
return {
|
||||
id: integration.id,
|
||||
name: integration.name,
|
||||
kind: integration.kind,
|
||||
url: integration.url,
|
||||
permissions: {
|
||||
hasUseAccess:
|
||||
permissions.includes("use") || permissions.includes("interact") || permissions.includes("full"),
|
||||
hasInteractAccess: permissions.includes("interact") || permissions.includes("full"),
|
||||
hasFullAccess: permissions.includes("full"),
|
||||
},
|
||||
permissions,
|
||||
};
|
||||
})
|
||||
.toSorted(
|
||||
@@ -138,21 +131,14 @@ export const integrationRouter = createTRPCRouter({
|
||||
});
|
||||
return integrationsFromDb
|
||||
.map((integration) => {
|
||||
const permissions = integration.userPermissions
|
||||
.map(({ permission }) => permission)
|
||||
.concat(integration.groupPermissions.map(({ permission }) => permission));
|
||||
const permissions = constructIntegrationPermissions(integration, ctx.session);
|
||||
|
||||
return {
|
||||
id: integration.id,
|
||||
name: integration.name,
|
||||
kind: integration.kind,
|
||||
url: integration.url,
|
||||
permissions: {
|
||||
hasUseAccess:
|
||||
permissions.includes("use") || permissions.includes("interact") || permissions.includes("full"),
|
||||
hasInteractAccess: permissions.includes("interact") || permissions.includes("full"),
|
||||
hasFullAccess: permissions.includes("full"),
|
||||
},
|
||||
permissions,
|
||||
};
|
||||
})
|
||||
.toSorted(
|
||||
@@ -189,21 +175,14 @@ export const integrationRouter = createTRPCRouter({
|
||||
});
|
||||
return integrationsFromDb
|
||||
.map((integration) => {
|
||||
const permissions = integration.userPermissions
|
||||
.map(({ permission }) => permission)
|
||||
.concat(integration.groupPermissions.map(({ permission }) => permission));
|
||||
const permissions = constructIntegrationPermissions(integration, ctx.session);
|
||||
|
||||
return {
|
||||
id: integration.id,
|
||||
name: integration.name,
|
||||
kind: integration.kind,
|
||||
url: integration.url,
|
||||
permissions: {
|
||||
hasUseAccess:
|
||||
permissions.includes("use") || permissions.includes("interact") || permissions.includes("full"),
|
||||
hasInteractAccess: permissions.includes("interact") || permissions.includes("full"),
|
||||
hasFullAccess: permissions.includes("full"),
|
||||
},
|
||||
permissions,
|
||||
};
|
||||
})
|
||||
.toSorted(
|
||||
|
||||
@@ -42,10 +42,10 @@ describe("KubernetesClientRegistry", () => {
|
||||
for (const contextId of ["home", "remote", "metrics-degraded"]) {
|
||||
const client = registry.getClient(contextId);
|
||||
vi.spyOn(client.versionApi, "getCode").mockResolvedValue({} as never);
|
||||
vi.spyOn(client.metricsApi, "getNodeMetrics").mockResolvedValue({ items: [] } as never);
|
||||
vi.spyOn(client.metricsProbeApi, "listClusterCustomObject").mockResolvedValue({ items: [] } as never);
|
||||
}
|
||||
registry.getClient("remote").versionApi.getCode = vi.fn().mockRejectedValue(new Error("offline"));
|
||||
registry.getClient("metrics-degraded").metricsApi.getNodeMetrics = vi
|
||||
registry.getClient("metrics-degraded").metricsProbeApi.listClusterCustomObject = vi
|
||||
.fn()
|
||||
.mockRejectedValue(new Error("no metrics"));
|
||||
|
||||
@@ -65,6 +65,40 @@ describe("KubernetesClientRegistry", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
test("times out an unresponsive context without delaying healthy contexts", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const registry = createRegistry();
|
||||
for (const contextId of ["home", "remote", "metrics-degraded"]) {
|
||||
const client = registry.getClient(contextId);
|
||||
vi.spyOn(client.versionApi, "getCode").mockResolvedValue({} as never);
|
||||
vi.spyOn(client.metricsProbeApi, "listClusterCustomObject").mockResolvedValue({ items: [] } as never);
|
||||
}
|
||||
let observedSignal: AbortSignal | undefined;
|
||||
vi.spyOn(registry.getClient("remote").versionApi, "getCode").mockImplementation(async (_request, options) => {
|
||||
const middleware = (
|
||||
options as { middleware?: { pre?: (request: unknown) => { toPromise: () => Promise<unknown> } }[] }
|
||||
).middleware?.[0];
|
||||
await middleware?.pre?.({ setSignal: (signal: AbortSignal) => (observedSignal = signal) }).toPromise();
|
||||
return await new Promise<never>(() => undefined);
|
||||
});
|
||||
|
||||
const resultPromise = registry.getContextsAsync(50);
|
||||
await vi.advanceTimersByTimeAsync(50);
|
||||
const result = await resultPromise;
|
||||
|
||||
expect(observedSignal?.aborted).toBe(true);
|
||||
expect(result.contexts).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ contextId: "home", status: "available" }),
|
||||
expect.objectContaining({ contextId: "remote", status: "unavailable" }),
|
||||
]),
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
test("rejects unknown context IDs", () => {
|
||||
expect(() => createRegistry().getClient("missing")).toThrow(KubernetesContextNotFoundError);
|
||||
});
|
||||
@@ -80,7 +114,7 @@ describe("KubernetesClientRegistry", () => {
|
||||
const registry = new KubernetesClientRegistry(kubeConfig);
|
||||
const client = registry.getClient("first");
|
||||
vi.spyOn(client.versionApi, "getCode").mockResolvedValue({} as never);
|
||||
vi.spyOn(client.metricsApi, "getNodeMetrics").mockResolvedValue({ items: [] } as never);
|
||||
vi.spyOn(client.metricsProbeApi, "listClusterCustomObject").mockResolvedValue({ items: [] } as never);
|
||||
|
||||
const result = await registry.getContextsAsync();
|
||||
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import * as fs from "fs";
|
||||
import { CoreV1Api, KubeConfig, Metrics, NetworkingV1Api, VersionApi } from "@kubernetes/client-node";
|
||||
import type { RequestContext, ResponseContext } from "@kubernetes/client-node";
|
||||
import { CoreV1Api, CustomObjectsApi, KubeConfig, Metrics, NetworkingV1Api, VersionApi } from "@kubernetes/client-node";
|
||||
import { of } from "@kubernetes/client-node/dist/gen/rxjsStub.js";
|
||||
|
||||
import { env } from "../../env";
|
||||
|
||||
@@ -11,12 +13,50 @@ export interface KubernetesContextStatus {
|
||||
isDefault: boolean;
|
||||
}
|
||||
|
||||
export const kubernetesContextProbeTimeoutMs = 5_000;
|
||||
|
||||
const withKubernetesProbeTimeoutAsync = async <T>(
|
||||
operation: (signal: AbortSignal) => Promise<T>,
|
||||
timeoutMs: number,
|
||||
) => {
|
||||
const controller = new AbortController();
|
||||
const timeoutError = new Error("Kubernetes context probe timed out");
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
|
||||
try {
|
||||
return await Promise.race([
|
||||
operation(controller.signal),
|
||||
new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
controller.abort(timeoutError);
|
||||
reject(timeoutError);
|
||||
}, timeoutMs);
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
};
|
||||
|
||||
const abortableKubernetesRequest = (signal: AbortSignal) => ({
|
||||
middleware: [
|
||||
{
|
||||
pre: (request: RequestContext) => {
|
||||
request.setSignal(signal);
|
||||
return of(request);
|
||||
},
|
||||
post: (response: ResponseContext) => of(response),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
export class KubernetesClient {
|
||||
private static registry: KubernetesClientRegistry | null = null;
|
||||
public kubeConfig: KubeConfig;
|
||||
public coreApi: CoreV1Api;
|
||||
public networkingApi: NetworkingV1Api;
|
||||
public metricsApi: Metrics;
|
||||
public metricsProbeApi: CustomObjectsApi;
|
||||
public versionApi: VersionApi;
|
||||
|
||||
public constructor(kubeConfig: KubeConfig) {
|
||||
@@ -24,6 +64,7 @@ export class KubernetesClient {
|
||||
this.coreApi = kubeConfig.makeApiClient(CoreV1Api);
|
||||
this.networkingApi = kubeConfig.makeApiClient(NetworkingV1Api);
|
||||
this.metricsApi = new Metrics(kubeConfig);
|
||||
this.metricsProbeApi = kubeConfig.makeApiClient(CustomObjectsApi);
|
||||
this.versionApi = kubeConfig.makeApiClient(VersionApi);
|
||||
}
|
||||
|
||||
@@ -76,12 +117,22 @@ export class KubernetesClientRegistry {
|
||||
return this.getClient(this.defaultContextId);
|
||||
}
|
||||
|
||||
public async getContextsAsync() {
|
||||
public async getContextsAsync(timeoutMs = kubernetesContextProbeTimeoutMs) {
|
||||
const contexts = await Promise.all(
|
||||
[...this.clients.entries()].map(async ([contextId, client]): Promise<KubernetesContextStatus> => {
|
||||
const [api, metrics] = await Promise.allSettled([
|
||||
client.versionApi.getCode(),
|
||||
client.metricsApi.getNodeMetrics(),
|
||||
withKubernetesProbeTimeoutAsync(
|
||||
async (signal) => await client.versionApi.getCode(undefined, abortableKubernetesRequest(signal)),
|
||||
timeoutMs,
|
||||
),
|
||||
withKubernetesProbeTimeoutAsync(
|
||||
async (signal) =>
|
||||
await client.metricsProbeApi.listClusterCustomObject(
|
||||
{ group: "metrics.k8s.io", version: "v1beta1", plural: "nodes" },
|
||||
abortableKubernetesRequest(signal),
|
||||
),
|
||||
timeoutMs,
|
||||
),
|
||||
]);
|
||||
const status =
|
||||
api.status === "rejected" ? "unavailable" : metrics.status === "rejected" ? "degraded" : "available";
|
||||
|
||||
@@ -418,6 +418,40 @@ test("prefers an explicitly linked app over duplicate app matches", async () =>
|
||||
expect(result.candidates[0]?.state).toBe("linked");
|
||||
});
|
||||
|
||||
test("marks a linked integration as moved when only its app URL still matches", async () => {
|
||||
getDockerDataAsyncMock.mockResolvedValueOnce({
|
||||
data: {
|
||||
containers: [createInventoryContainer({ endpointId: "home", id: "one", name: "sonarr", publicPort: 8989 })],
|
||||
endpoints: [{ id: "home", name: "Home", status: "available" }],
|
||||
},
|
||||
timestamp: new Date(),
|
||||
});
|
||||
findIntegrationsAsyncMock.mockResolvedValueOnce([
|
||||
{
|
||||
id: "integration",
|
||||
name: "Sonarr",
|
||||
kind: "sonarr",
|
||||
url: "http://old.example:8989",
|
||||
appId: "linked",
|
||||
items: [],
|
||||
secrets: [],
|
||||
},
|
||||
]);
|
||||
findAppsAsyncMock.mockResolvedValueOnce([
|
||||
{ id: "linked", name: "Sonarr", href: "http://home.example:8989", iconUrl: "" },
|
||||
]);
|
||||
|
||||
const result = await createAdminCaller().reconcileServices();
|
||||
|
||||
expect(result.candidates[0]?.state).toBe("moved");
|
||||
expect(result.candidates[0]?.nextAction).toBe("reviewIntegration");
|
||||
expect(result.candidates[0]?.representation.signals).toMatchObject({
|
||||
exactIntegrationUrl: false,
|
||||
exactAppUrl: true,
|
||||
linked: true,
|
||||
});
|
||||
});
|
||||
|
||||
test("does not offer to create an app when existing app matches are ambiguous", async () => {
|
||||
const dockerData = {
|
||||
data: {
|
||||
|
||||
@@ -56,6 +56,7 @@ describe("all should return all integrations", () => {
|
||||
expect(result.length).toBe(2);
|
||||
expect(result[0]!.kind).toBe("plex");
|
||||
expect(result[1]!.kind).toBe("homeAssistant");
|
||||
expect(result.every(({ permissions }) => permissions.hasFullAccess && permissions.hasUseAccess)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
Reference in new issue
Block a user