Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions __tests__/e2e/ci-mac-linux.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,14 @@ cd model
pip install -r requirements.txt
export fc_component_function_name=model-$(uname)-$(uname -m)-$RANDSTR-$RANDOM
python -u deploy_and_test_model.py --model-id iic/cv_LightweightEdge_ocr-recognitoin-general_damo --region cn-shanghai --auto-cleanup
sleep 5
sleep 10
python -u deploy_and_test_model.py --model-id Qwen/Qwen2.5-0.5B-Instruct --region cn-shanghai --auto-cleanup
sleep 5
sleep 10
python -u deploy_and_test_model.py --model-id iic/cv_LightweightEdge_ocr-recognitoin-general_damo --region cn-shanghai --storage oss --auto-cleanup
sleep 5
sleep 10
python -u deploy_and_test_model.py --model-id Qwen/Qwen2.5-0.5B-Instruct --region cn-shanghai --storage oss --auto-cleanup

sleep 5
sleep 10
echo "test model s_file.yaml"
# python -u test.py
s model download -t s_file.yaml
Expand Down
75 changes: 72 additions & 3 deletions __tests__/ut/commands/deploy/impl/provision_config_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,27 @@ describe('ProvisionConfig', () => {
});
expect(provisionConfig.ProvisionMode).toBe('async');
});

it('should initialize correctly with drain mode config', () => {
const inputsWithMode = {
...mockInputs,
props: {
...mockInputs.props,
provisionConfig: {
defaultTarget: 10,
alwaysAllocateCPU: false,
alwaysAllocateGPU: false,
scheduledActions: [],
targetTrackingPolicies: [],
mode: 'drain',
},
},
};

provisionConfig = new ProvisionConfig(inputsWithMode, mockOpts);

expect(provisionConfig.ProvisionMode).toBe('drain');
});
});

describe('before', () => {
Expand Down Expand Up @@ -350,12 +371,15 @@ describe('ProvisionConfig', () => {
});

describe('waitForProvisionReady', () => {
it('should return immediately when target is 0', async () => {
it('should still call getFunctionProvisionConfig when target is 0 but loop exits early', async () => {
provisionConfig = new ProvisionConfig(mockInputs, mockOpts);

// Mock fcSdk
const mockFcSdk = {
getFunctionProvisionConfig: jest.fn(),
getFunctionProvisionConfig: jest.fn().mockImplementation(() => {
// 返回模拟数据,但我们会验证它被调用了
return Promise.resolve({ current: 0, target: 0 });
}),
disableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
enableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
};
Expand All @@ -366,7 +390,8 @@ describe('ProvisionConfig', () => {

await (provisionConfig as any).waitForProvisionReady('LATEST', { target: 0 });

expect(provisionConfig.fcSdk.getFunctionProvisionConfig).not.toHaveBeenCalled();
// 验证getFunctionProvisionConfig至少被调用了一次
expect(provisionConfig.fcSdk.getFunctionProvisionConfig).toHaveBeenCalled();
});

it('should wait until current equals target', async () => {
Expand Down Expand Up @@ -474,6 +499,7 @@ describe('ProvisionConfig', () => {

// Mock fcSdk
const mockFcSdk = {
getFunctionProvisionConfig: jest.fn(),
disableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
enableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
};
Expand All @@ -499,6 +525,7 @@ describe('ProvisionConfig', () => {

// Mock fcSdk
const mockFcSdk = {
getFunctionProvisionConfig: jest.fn(),
disableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
enableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
};
Expand All @@ -518,6 +545,32 @@ describe('ProvisionConfig', () => {
expect(provisionConfig.fcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function');
});

it('should handle drain mode with target 0', async () => {
provisionConfig = new ProvisionConfig(mockInputs, mockOpts);
provisionConfig.ProvisionMode = 'drain';

// Mock fcSdk
const mockFcSdk = {
getFunctionProvisionConfig: jest.fn(),
disableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
enableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
};
Object.defineProperty(provisionConfig, 'fcSdk', {
value: mockFcSdk,
writable: true,
});

await (provisionConfig as any).waitForProvisionReady('LATEST', { target: 0 });

expect(provisionConfig.fcSdk.disableFunctionInvocation).toHaveBeenCalledWith(
'test-function',
true,
'Fast scale-to-zero',
);
expect(sleepMock).toHaveBeenCalledWith(5);
expect(provisionConfig.fcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function');
});

it('should handle error retries in waitForProvisionReady', async () => {
provisionConfig = new ProvisionConfig(mockInputs, mockOpts);

Expand Down Expand Up @@ -753,6 +806,22 @@ describe('ProvisionConfig', () => {
});
});

it('should keep target when defaultTarget does not exist', () => {
provisionConfig = new ProvisionConfig(mockInputs, mockOpts);

const config = {
target: 5,
current: 5,
functionArn: 'arn:xxx',
};

const result = (provisionConfig as any).sanitizeProvisionConfig(config);

expect(result).toEqual({
target: 5,
});
});

it('should remove empty targetTrackingPolicies', () => {
provisionConfig = new ProvisionConfig(mockInputs, mockOpts);

Expand Down
46 changes: 43 additions & 3 deletions __tests__/ut/commands/deploy/impl/scaling_config_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -322,6 +322,17 @@ describe('ScalingConfig', () => {
});
expect(scalingConfig.ScalingMode).toBe('async');
});

it('should initialize correctly with drain mode config', () => {
mockInputs.props.scalingConfig = {
minInstances: 1,
mode: 'drain',
} as any;

scalingConfig = new ScalingConfig(mockInputs, mockOpts);

expect(scalingConfig.ScalingMode).toBe('drain');
});
});

describe('before', () => {
Expand Down Expand Up @@ -611,12 +622,14 @@ describe('ScalingConfig', () => {
});

describe('waitForScalingReady', () => {
it('should return immediately when minInstances is 0', async () => {
it('should still call getFunctionScalingConfig when minInstances is 0 but loop exits early', async () => {
scalingConfig = new ScalingConfig(mockInputs, mockOpts);

// Mock fcSdk
const mockFcSdk = {
getFunctionScalingConfig: jest.fn(),
getFunctionScalingConfig: jest
.fn()
.mockResolvedValue({ currentInstances: 0, minInstances: 0 }),
disableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
enableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
};
Expand All @@ -627,7 +640,8 @@ describe('ScalingConfig', () => {

await (scalingConfig as any).waitForScalingReady('LATEST', { minInstances: 0 });

expect(mockFcSdk.getFunctionScalingConfig).not.toHaveBeenCalled();
// 验证getFunctionScalingConfig至少被调用了一次
expect(mockFcSdk.getFunctionScalingConfig).toHaveBeenCalled();
});

it('should wait until currentInstances reaches minInstances', async () => {
Expand Down Expand Up @@ -681,6 +695,32 @@ describe('ScalingConfig', () => {
expect(mockFcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function');
});

it('should handle drain mode without minInstances', async () => {
scalingConfig = new ScalingConfig(mockInputs, mockOpts);
scalingConfig.ScalingMode = 'drain';

// Mock fcSdk
const mockFcSdk = {
disableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
enableFunctionInvocation: jest.fn().mockResolvedValue(undefined),
getFunctionScalingConfig: jest.fn().mockResolvedValue({}),
};
Object.defineProperty(scalingConfig, 'fcSdk', {
value: mockFcSdk,
writable: true,
});

await (scalingConfig as any).waitForScalingReady('LATEST', {});

expect(mockFcSdk.disableFunctionInvocation).toHaveBeenCalledWith(
'test-function',
true,
'Fast scale-to-zero',
);
expect(sleepMock).toHaveBeenCalledWith(5);
expect(mockFcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function');
});

it('should timeout when max retries reached', async () => {
scalingConfig = new ScalingConfig(mockInputs, mockOpts);

Expand Down
2 changes: 1 addition & 1 deletion publish.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ Type: Component
Name: fc3
Provider:
- 阿里云
Version: 0.1.8
Version: 0.1.9
Description: 阿里云函数计算全生命周期管理
HomePage: https://github.com/devsapp/fc3
Organization: 阿里云函数计算(FC)
Expand Down
51 changes: 39 additions & 12 deletions src/resources/fc/error-code.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,20 +53,47 @@ export const isFunctionStateWaitTimedOut = (ex) => {
return false;
};

const PROVISION_ERROR_CODES = [
FC_API_ERROR_CODE.ProvisionConfigExist,
FC_API_ERROR_CODE.ResidentScalingConfigExists,
];

const ERROR_MESSAGES = {
GPU_TYPE_CHANGE: 'GPU type should not be changed with resident scaling config',
GPU_TYPE_MISMATCH: (localGPUType, remoteGPUType) =>
`function gpu type '${localGPUType}' doesn't match resident pool gpu type '${remoteGPUType}'`,
IDLE_PROVISION_CONFIG: (functionName) =>
`idle provision config exists for function '${functionName}'`,
FUNCTION_BOUND_TO_POOL: (functionName) =>
`Function '${functionName}' has been bound to resident source pool`,
};

const checkProvisionErrorCode = (ex) => {
const code = ex?.code;
return code && PROVISION_ERROR_CODES.includes(code);
};

const checkInvalidArgumentConditions = (ex, localGPUType, remoteGPUType, functionName) => {
if (!isInvalidArgument(ex)) return false;

const message = ex?.message;
const code = ex?.code;

return (
message?.includes(ERROR_MESSAGES.GPU_TYPE_CHANGE) ||
message?.includes(ERROR_MESSAGES.GPU_TYPE_MISMATCH(localGPUType, remoteGPUType)) ||
message?.includes(ERROR_MESSAGES.IDLE_PROVISION_CONFIG(functionName)) ||
code === FC_API_ERROR_CODE.ResourcePoolInsufficientCapacity ||
message?.includes(ERROR_MESSAGES.FUNCTION_BOUND_TO_POOL(functionName))
);
};

export const isFunctionScalingConfigError = (
ex,
{ localGPUType = '', remoteGPUType = '', functionName = 'F' },
) => {
if (
isInvalidArgument(ex) &&
(ex.message.includes('GPU type should not be changed with resident scaling config') ||
ex.message.includes(
`function gpu type '${localGPUType}' doesn't match resident pool gpu type '${remoteGPUType}'`,
) ||
ex.message.includes(`idle provision config exists for function '${functionName}'`) ||
ex.code === FC_API_ERROR_CODE.ResourcePoolInsufficientCapacity)
) {
return true;
}
return false;
return (
checkProvisionErrorCode(ex) ||
checkInvalidArgumentConditions(ex, localGPUType, remoteGPUType, functionName)
);
};
14 changes: 8 additions & 6 deletions src/subCommands/deploy/impl/provision_config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,12 @@ export default class ProvisionConfig extends Base {

// 如果没有目标值或目标值为0,则无需等待
if (!realTarget || realTarget <= 0) {
if (this.ProvisionMode !== 'drain') {
return;
} else {
if (this.ProvisionMode === 'drain') {
logger.info(`disableFunctionInvocation ${this.functionName} ...`);
await this.fcSdk.disableFunctionInvocation(this.functionName, true, 'Fast scale-to-zero');
await sleep(5);
logger.info(`enableFunctionInvocation ${this.functionName} ...`);
await this.fcSdk.enableFunctionInvocation(this.functionName);
return;
}
}

Expand All @@ -114,7 +111,7 @@ export default class ProvisionConfig extends Base {
const { current, currentError } = result || {};

// 检查是否已达到目标值
if (current && current === realTarget) {
if (current === undefined || (current && current === realTarget)) {
logger.info(
`ProvisionConfig of ${this.functionName}/${qualifier} is ready. Current: ${current}, Target: ${realTarget}`,
);
Expand All @@ -138,7 +135,12 @@ export default class ProvisionConfig extends Base {
logger.warn('=========== LAST 10 MINUTES END =========== ');
*/
// 如果是系统内部错误,则继续尝试
if (!currentError.includes('an internal error has occurred')) {
if (
!(
currentError.includes('an internal error has occurred') ||
currentError.includes('Resources are being replenished')
)
) {
// 不是系统内部错误,满足一定的重试次数则退出
getCurrentErrorCount++;
if (getCurrentErrorCount > 3 || (index > 6 && getCurrentErrorCount > 0)) {
Expand Down
20 changes: 14 additions & 6 deletions src/subCommands/deploy/impl/scaling_config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -90,15 +90,12 @@ export default class ScalingConfig extends Base {

// 如果没有最小实例数或最小实例数为0,则无需等待
if (!minInstances || minInstances <= 0) {
if (this.ScalingMode !== 'drain') {
return;
} else {
if (this.ScalingMode === 'drain') {
logger.info(`disableFunctionInvocation ${this.functionName} ...`);
await this.fcSdk.disableFunctionInvocation(this.functionName, true, 'Fast scale-to-zero');
await sleep(5);
logger.info(`enableFunctionInvocation ${this.functionName} ...`);
await this.fcSdk.enableFunctionInvocation(this.functionName);
return;
}
}

Expand All @@ -110,16 +107,27 @@ export default class ScalingConfig extends Base {
const result = await this.fcSdk.getFunctionScalingConfig(this.functionName, qualifier);
const { currentInstances, currentError } = result || {};

logger.debug(
`get ${this.functionName}/${qualifier} scaling config result: ${JSON.stringify(result)}`,
);
// 检查是否已达到最小实例数
if (currentInstances && currentInstances >= result.minInstances) {
if (
currentInstances === undefined ||
(currentInstances && currentInstances === result.minInstances)
) {
logger.info(
`ScalingConfig of ${this.functionName}/${qualifier} is ready. CurrentInstances: ${currentInstances}, MinInstances: ${minInstances}`,
);
return;
}
if (currentError && currentError.length > 0) {
// 如果是系统内部错误,则继续尝试
if (!currentError.includes('an internal error has occurred')) {
if (
!(
currentError.includes('an internal error has occurred') ||
currentError.includes('Resources are being replenished')
)
) {
// 不是系统内部错误,满足一定的重试次数则退出
getCurrentErrorCount++;
if (getCurrentErrorCount > 3 || (index > 6 && getCurrentErrorCount > 0)) {
Expand Down
Loading