diff --git a/__tests__/e2e/ci-mac-linux.sh b/__tests__/e2e/ci-mac-linux.sh index 63923787..e4cea486 100755 --- a/__tests__/e2e/ci-mac-linux.sh +++ b/__tests__/e2e/ci-mac-linux.sh @@ -32,14 +32,14 @@ cd model pip install -r requirements.txt export fc_component_function_name=model-$(uname)-$(uname -m)-$RANDSTR-$RANDOM python -u deploy_and_test_model.py --model-id iic/cv_LightweightEdge_ocr-recognitoin-general_damo --region cn-shanghai --auto-cleanup -sleep 5 +sleep 10 python -u deploy_and_test_model.py --model-id Qwen/Qwen2.5-0.5B-Instruct --region cn-shanghai --auto-cleanup -sleep 5 +sleep 10 python -u deploy_and_test_model.py --model-id iic/cv_LightweightEdge_ocr-recognitoin-general_damo --region cn-shanghai --storage oss --auto-cleanup -sleep 5 +sleep 10 python -u deploy_and_test_model.py --model-id Qwen/Qwen2.5-0.5B-Instruct --region cn-shanghai --storage oss --auto-cleanup -sleep 5 +sleep 10 echo "test model s_file.yaml" # python -u test.py s model download -t s_file.yaml diff --git a/__tests__/ut/commands/deploy/impl/provision_config_test.ts b/__tests__/ut/commands/deploy/impl/provision_config_test.ts index c37200e4..b3ebe8f9 100644 --- a/__tests__/ut/commands/deploy/impl/provision_config_test.ts +++ b/__tests__/ut/commands/deploy/impl/provision_config_test.ts @@ -100,6 +100,27 @@ describe('ProvisionConfig', () => { }); expect(provisionConfig.ProvisionMode).toBe('async'); }); + + it('should initialize correctly with drain mode config', () => { + const inputsWithMode = { + ...mockInputs, + props: { + ...mockInputs.props, + provisionConfig: { + defaultTarget: 10, + alwaysAllocateCPU: false, + alwaysAllocateGPU: false, + scheduledActions: [], + targetTrackingPolicies: [], + mode: 'drain', + }, + }, + }; + + provisionConfig = new ProvisionConfig(inputsWithMode, mockOpts); + + expect(provisionConfig.ProvisionMode).toBe('drain'); + }); }); describe('before', () => { @@ -350,12 +371,15 @@ describe('ProvisionConfig', () => { }); describe('waitForProvisionReady', () => { - it('should return immediately when target is 0', async () => { + it('should still call getFunctionProvisionConfig when target is 0 but loop exits early', async () => { provisionConfig = new ProvisionConfig(mockInputs, mockOpts); // Mock fcSdk const mockFcSdk = { - getFunctionProvisionConfig: jest.fn(), + getFunctionProvisionConfig: jest.fn().mockImplementation(() => { + // 返回模拟数据,但我们会验证它被调用了 + return Promise.resolve({ current: 0, target: 0 }); + }), disableFunctionInvocation: jest.fn().mockResolvedValue(undefined), enableFunctionInvocation: jest.fn().mockResolvedValue(undefined), }; @@ -366,7 +390,8 @@ describe('ProvisionConfig', () => { await (provisionConfig as any).waitForProvisionReady('LATEST', { target: 0 }); - expect(provisionConfig.fcSdk.getFunctionProvisionConfig).not.toHaveBeenCalled(); + // 验证getFunctionProvisionConfig至少被调用了一次 + expect(provisionConfig.fcSdk.getFunctionProvisionConfig).toHaveBeenCalled(); }); it('should wait until current equals target', async () => { @@ -474,6 +499,7 @@ describe('ProvisionConfig', () => { // Mock fcSdk const mockFcSdk = { + getFunctionProvisionConfig: jest.fn(), disableFunctionInvocation: jest.fn().mockResolvedValue(undefined), enableFunctionInvocation: jest.fn().mockResolvedValue(undefined), }; @@ -499,6 +525,7 @@ describe('ProvisionConfig', () => { // Mock fcSdk const mockFcSdk = { + getFunctionProvisionConfig: jest.fn(), disableFunctionInvocation: jest.fn().mockResolvedValue(undefined), enableFunctionInvocation: jest.fn().mockResolvedValue(undefined), }; @@ -518,6 +545,32 @@ describe('ProvisionConfig', () => { expect(provisionConfig.fcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function'); }); + it('should handle drain mode with target 0', async () => { + provisionConfig = new ProvisionConfig(mockInputs, mockOpts); + provisionConfig.ProvisionMode = 'drain'; + + // Mock fcSdk + const mockFcSdk = { + getFunctionProvisionConfig: jest.fn(), + disableFunctionInvocation: jest.fn().mockResolvedValue(undefined), + enableFunctionInvocation: jest.fn().mockResolvedValue(undefined), + }; + Object.defineProperty(provisionConfig, 'fcSdk', { + value: mockFcSdk, + writable: true, + }); + + await (provisionConfig as any).waitForProvisionReady('LATEST', { target: 0 }); + + expect(provisionConfig.fcSdk.disableFunctionInvocation).toHaveBeenCalledWith( + 'test-function', + true, + 'Fast scale-to-zero', + ); + expect(sleepMock).toHaveBeenCalledWith(5); + expect(provisionConfig.fcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function'); + }); + it('should handle error retries in waitForProvisionReady', async () => { provisionConfig = new ProvisionConfig(mockInputs, mockOpts); @@ -753,6 +806,22 @@ describe('ProvisionConfig', () => { }); }); + it('should keep target when defaultTarget does not exist', () => { + provisionConfig = new ProvisionConfig(mockInputs, mockOpts); + + const config = { + target: 5, + current: 5, + functionArn: 'arn:xxx', + }; + + const result = (provisionConfig as any).sanitizeProvisionConfig(config); + + expect(result).toEqual({ + target: 5, + }); + }); + it('should remove empty targetTrackingPolicies', () => { provisionConfig = new ProvisionConfig(mockInputs, mockOpts); diff --git a/__tests__/ut/commands/deploy/impl/scaling_config_test.ts b/__tests__/ut/commands/deploy/impl/scaling_config_test.ts index 73ebe5a6..ad23c5a7 100644 --- a/__tests__/ut/commands/deploy/impl/scaling_config_test.ts +++ b/__tests__/ut/commands/deploy/impl/scaling_config_test.ts @@ -322,6 +322,17 @@ describe('ScalingConfig', () => { }); expect(scalingConfig.ScalingMode).toBe('async'); }); + + it('should initialize correctly with drain mode config', () => { + mockInputs.props.scalingConfig = { + minInstances: 1, + mode: 'drain', + } as any; + + scalingConfig = new ScalingConfig(mockInputs, mockOpts); + + expect(scalingConfig.ScalingMode).toBe('drain'); + }); }); describe('before', () => { @@ -611,12 +622,14 @@ describe('ScalingConfig', () => { }); describe('waitForScalingReady', () => { - it('should return immediately when minInstances is 0', async () => { + it('should still call getFunctionScalingConfig when minInstances is 0 but loop exits early', async () => { scalingConfig = new ScalingConfig(mockInputs, mockOpts); // Mock fcSdk const mockFcSdk = { - getFunctionScalingConfig: jest.fn(), + getFunctionScalingConfig: jest + .fn() + .mockResolvedValue({ currentInstances: 0, minInstances: 0 }), disableFunctionInvocation: jest.fn().mockResolvedValue(undefined), enableFunctionInvocation: jest.fn().mockResolvedValue(undefined), }; @@ -627,7 +640,8 @@ describe('ScalingConfig', () => { await (scalingConfig as any).waitForScalingReady('LATEST', { minInstances: 0 }); - expect(mockFcSdk.getFunctionScalingConfig).not.toHaveBeenCalled(); + // 验证getFunctionScalingConfig至少被调用了一次 + expect(mockFcSdk.getFunctionScalingConfig).toHaveBeenCalled(); }); it('should wait until currentInstances reaches minInstances', async () => { @@ -681,6 +695,32 @@ describe('ScalingConfig', () => { expect(mockFcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function'); }); + it('should handle drain mode without minInstances', async () => { + scalingConfig = new ScalingConfig(mockInputs, mockOpts); + scalingConfig.ScalingMode = 'drain'; + + // Mock fcSdk + const mockFcSdk = { + disableFunctionInvocation: jest.fn().mockResolvedValue(undefined), + enableFunctionInvocation: jest.fn().mockResolvedValue(undefined), + getFunctionScalingConfig: jest.fn().mockResolvedValue({}), + }; + Object.defineProperty(scalingConfig, 'fcSdk', { + value: mockFcSdk, + writable: true, + }); + + await (scalingConfig as any).waitForScalingReady('LATEST', {}); + + expect(mockFcSdk.disableFunctionInvocation).toHaveBeenCalledWith( + 'test-function', + true, + 'Fast scale-to-zero', + ); + expect(sleepMock).toHaveBeenCalledWith(5); + expect(mockFcSdk.enableFunctionInvocation).toHaveBeenCalledWith('test-function'); + }); + it('should timeout when max retries reached', async () => { scalingConfig = new ScalingConfig(mockInputs, mockOpts); diff --git a/publish.yaml b/publish.yaml index 1785cd29..b588dbf0 100644 --- a/publish.yaml +++ b/publish.yaml @@ -3,7 +3,7 @@ Type: Component Name: fc3 Provider: - 阿里云 -Version: 0.1.8 +Version: 0.1.9 Description: 阿里云函数计算全生命周期管理 HomePage: https://github.com/devsapp/fc3 Organization: 阿里云函数计算(FC) diff --git a/src/resources/fc/error-code.ts b/src/resources/fc/error-code.ts index b237b18b..a2dd3d21 100644 --- a/src/resources/fc/error-code.ts +++ b/src/resources/fc/error-code.ts @@ -53,20 +53,47 @@ export const isFunctionStateWaitTimedOut = (ex) => { return false; }; +const PROVISION_ERROR_CODES = [ + FC_API_ERROR_CODE.ProvisionConfigExist, + FC_API_ERROR_CODE.ResidentScalingConfigExists, +]; + +const ERROR_MESSAGES = { + GPU_TYPE_CHANGE: 'GPU type should not be changed with resident scaling config', + GPU_TYPE_MISMATCH: (localGPUType, remoteGPUType) => + `function gpu type '${localGPUType}' doesn't match resident pool gpu type '${remoteGPUType}'`, + IDLE_PROVISION_CONFIG: (functionName) => + `idle provision config exists for function '${functionName}'`, + FUNCTION_BOUND_TO_POOL: (functionName) => + `Function '${functionName}' has been bound to resident source pool`, +}; + +const checkProvisionErrorCode = (ex) => { + const code = ex?.code; + return code && PROVISION_ERROR_CODES.includes(code); +}; + +const checkInvalidArgumentConditions = (ex, localGPUType, remoteGPUType, functionName) => { + if (!isInvalidArgument(ex)) return false; + + const message = ex?.message; + const code = ex?.code; + + return ( + message?.includes(ERROR_MESSAGES.GPU_TYPE_CHANGE) || + message?.includes(ERROR_MESSAGES.GPU_TYPE_MISMATCH(localGPUType, remoteGPUType)) || + message?.includes(ERROR_MESSAGES.IDLE_PROVISION_CONFIG(functionName)) || + code === FC_API_ERROR_CODE.ResourcePoolInsufficientCapacity || + message?.includes(ERROR_MESSAGES.FUNCTION_BOUND_TO_POOL(functionName)) + ); +}; + export const isFunctionScalingConfigError = ( ex, { localGPUType = '', remoteGPUType = '', functionName = 'F' }, ) => { - if ( - isInvalidArgument(ex) && - (ex.message.includes('GPU type should not be changed with resident scaling config') || - ex.message.includes( - `function gpu type '${localGPUType}' doesn't match resident pool gpu type '${remoteGPUType}'`, - ) || - ex.message.includes(`idle provision config exists for function '${functionName}'`) || - ex.code === FC_API_ERROR_CODE.ResourcePoolInsufficientCapacity) - ) { - return true; - } - return false; + return ( + checkProvisionErrorCode(ex) || + checkInvalidArgumentConditions(ex, localGPUType, remoteGPUType, functionName) + ); }; diff --git a/src/subCommands/deploy/impl/provision_config.ts b/src/subCommands/deploy/impl/provision_config.ts index 11ed767c..d8e11514 100644 --- a/src/subCommands/deploy/impl/provision_config.ts +++ b/src/subCommands/deploy/impl/provision_config.ts @@ -93,15 +93,12 @@ export default class ProvisionConfig extends Base { // 如果没有目标值或目标值为0,则无需等待 if (!realTarget || realTarget <= 0) { - if (this.ProvisionMode !== 'drain') { - return; - } else { + if (this.ProvisionMode === 'drain') { logger.info(`disableFunctionInvocation ${this.functionName} ...`); await this.fcSdk.disableFunctionInvocation(this.functionName, true, 'Fast scale-to-zero'); await sleep(5); logger.info(`enableFunctionInvocation ${this.functionName} ...`); await this.fcSdk.enableFunctionInvocation(this.functionName); - return; } } @@ -114,7 +111,7 @@ export default class ProvisionConfig extends Base { const { current, currentError } = result || {}; // 检查是否已达到目标值 - if (current && current === realTarget) { + if (current === undefined || (current && current === realTarget)) { logger.info( `ProvisionConfig of ${this.functionName}/${qualifier} is ready. Current: ${current}, Target: ${realTarget}`, ); @@ -138,7 +135,12 @@ export default class ProvisionConfig extends Base { logger.warn('=========== LAST 10 MINUTES END =========== '); */ // 如果是系统内部错误,则继续尝试 - if (!currentError.includes('an internal error has occurred')) { + if ( + !( + currentError.includes('an internal error has occurred') || + currentError.includes('Resources are being replenished') + ) + ) { // 不是系统内部错误,满足一定的重试次数则退出 getCurrentErrorCount++; if (getCurrentErrorCount > 3 || (index > 6 && getCurrentErrorCount > 0)) { diff --git a/src/subCommands/deploy/impl/scaling_config.ts b/src/subCommands/deploy/impl/scaling_config.ts index 841fdbeb..245c3c4c 100644 --- a/src/subCommands/deploy/impl/scaling_config.ts +++ b/src/subCommands/deploy/impl/scaling_config.ts @@ -90,15 +90,12 @@ export default class ScalingConfig extends Base { // 如果没有最小实例数或最小实例数为0,则无需等待 if (!minInstances || minInstances <= 0) { - if (this.ScalingMode !== 'drain') { - return; - } else { + if (this.ScalingMode === 'drain') { logger.info(`disableFunctionInvocation ${this.functionName} ...`); await this.fcSdk.disableFunctionInvocation(this.functionName, true, 'Fast scale-to-zero'); await sleep(5); logger.info(`enableFunctionInvocation ${this.functionName} ...`); await this.fcSdk.enableFunctionInvocation(this.functionName); - return; } } @@ -110,8 +107,14 @@ export default class ScalingConfig extends Base { const result = await this.fcSdk.getFunctionScalingConfig(this.functionName, qualifier); const { currentInstances, currentError } = result || {}; + logger.debug( + `get ${this.functionName}/${qualifier} scaling config result: ${JSON.stringify(result)}`, + ); // 检查是否已达到最小实例数 - if (currentInstances && currentInstances >= result.minInstances) { + if ( + currentInstances === undefined || + (currentInstances && currentInstances === result.minInstances) + ) { logger.info( `ScalingConfig of ${this.functionName}/${qualifier} is ready. CurrentInstances: ${currentInstances}, MinInstances: ${minInstances}`, ); @@ -119,7 +122,12 @@ export default class ScalingConfig extends Base { } if (currentError && currentError.length > 0) { // 如果是系统内部错误,则继续尝试 - if (!currentError.includes('an internal error has occurred')) { + if ( + !( + currentError.includes('an internal error has occurred') || + currentError.includes('Resources are being replenished') + ) + ) { // 不是系统内部错误,满足一定的重试次数则退出 getCurrentErrorCount++; if (getCurrentErrorCount > 3 || (index > 6 && getCurrentErrorCount > 0)) {