diff --git a/src/endpoints/tokens/token.service.ts b/src/endpoints/tokens/token.service.ts index b9a3b9f41..4ec0f8349 100644 --- a/src/endpoints/tokens/token.service.ts +++ b/src/endpoints/tokens/token.service.ts @@ -761,9 +761,30 @@ export class TokenService { } this.logger.log(`Starting to fetch all tokens`); + let tokens = await this.fetchAllTokensWithoutDetails(); + + await this.applyTokenDetails(tokens); + + await this.applyTokensCreatedDuringProcessing(tokens); + + this.logger.log(`Sorting and finalizing ${tokens.length} tokens`); + tokens = tokens.sortedDescending( + token => token.assets ? 1 : 0, + token => token.marketCap ? 1 : 0, + token => token.isLowLiquidity || token.assets?.priceSource?.type === TokenAssetsPriceSourceType.customUrl ? 0 : (token.marketCap ?? 0), + token => token.transactions ?? 0, + ); + + tokens = [...tokens, await this.buildEgldToken()]; + + this.logger.log(`Total tokens processed: ${tokens.length}`); + return tokens; + } + + private async fetchAllTokensWithoutDetails(): Promise { const startFungible = Date.now(); const tokensProperties = await this.esdtService.getAllFungibleTokenProperties(); - let tokens = tokensProperties.map(properties => ApiUtils.mergeObjects(new TokenDetailed(), properties)); + const tokens = tokensProperties.map(properties => ApiUtils.mergeObjects(new TokenDetailed(), properties)); this.logger.log(`Fetched ${tokens.length} fungible tokens in ${Date.now() - startFungible}ms`); const allAssets = await this.assetsService.getAllTokenAssets(); @@ -785,20 +806,32 @@ export class TokenService { tokens.push(this.buildMetaEsdtToken(collection)); } - await this.applyTokenDetails(tokens); + return tokens; + } - this.logger.log(`Sorting and finalizing ${tokens.length} tokens`); - tokens = tokens.sortedDescending( - token => token.assets ? 1 : 0, - token => token.marketCap ? 1 : 0, - token => token.isLowLiquidity || token.assets?.priceSource?.type === TokenAssetsPriceSourceType.customUrl ? 0 : (token.marketCap ?? 0), - token => token.transactions ?? 0, - ); + private async applyTokensCreatedDuringProcessing(tokens: TokenDetailed[]): Promise { + // processing all tokens takes tens of seconds, so re-fetch the token list and + // process only the tokens created in the meantime, instead of waiting for the next refresh + try { + const startFetchAndProcess = Date.now(); + const processedIdentifiers = new Set(tokens.map(token => token.identifier)); + const latestTokens = await this.fetchAllTokensWithoutDetails(); + const newTokens = latestTokens.filter(token => !processedIdentifiers.has(token.identifier)); - tokens = [...tokens, await this.buildEgldToken()]; + if (newTokens.length === 0) { + return; + } - this.logger.log(`Total tokens processed: ${tokens.length}`); - return tokens; + await this.applyTokenDetails(newTokens); + + tokens.push(...newTokens); + + const endFetchAndProcess = Date.now(); + this.logger.log(`Processed ${newTokens.length} tokens created while processing all tokens in ${endFetchAndProcess - startFetchAndProcess}ms`); + } catch (error) { + this.logger.error('Could not apply tokens created while processing all tokens'); + this.logger.error(error); + } } async getTokenRaw(rawIdentifier: string): Promise { diff --git a/src/test/unit/services/tokens.spec.ts b/src/test/unit/services/tokens.spec.ts index 6e7ea9488..fb4ebedc2 100644 --- a/src/test/unit/services/tokens.spec.ts +++ b/src/test/unit/services/tokens.spec.ts @@ -723,7 +723,8 @@ describe('Token Service', () => { expect(apiConfigService.isTokensFetchFeatureEnabled).toHaveBeenCalled(); expect(esdtService.getAllFungibleTokenProperties).toHaveBeenCalled(); - expect(assetsService.getAllTokenAssets).toHaveBeenCalledTimes(1); + // the token list is fetched once for processing and once more to catch tokens created in the meantime + expect(assetsService.getAllTokenAssets).toHaveBeenCalledTimes(2); mockTokens.forEach(mockToken => { mockToken.name = mockTokenAssets.name; @@ -797,6 +798,67 @@ describe('Token Service', () => { expect(egldToken?.supply).toBe('0'); expect(egldToken?.circulatingSupply).toBe('0'); }); + + describe('tokens created while processing all tokens', () => { + const mockTokenSupply: Partial = { totalSupply: '1000', circulatingSupply: '1000' }; + + beforeEach(() => { + jest.spyOn(apiConfigService, 'isTokensFetchFeatureEnabled').mockReturnValue(false); + jest.spyOn(assetsService, 'getAllTokenAssets').mockResolvedValue({}); + jest.spyOn(assetsService, 'getTokenAssets').mockResolvedValue(undefined); + + jest.spyOn(tokenService as any, 'batchProcessTokens').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexLiquidity').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexPrices').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexPairType').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexPairTradesCount').mockImplementation(() => Promise.resolve()); + jest.spyOn(cacheService as any, 'batchApplyAll').mockImplementation(() => Promise.resolve()); + jest.spyOn(dataApiService, 'getEsdtTokenPrice').mockResolvedValue(undefined); + jest.spyOn(dataApiService, 'getEgldPrice').mockResolvedValue(100); + jest.spyOn(esdtService, 'getTokenSupply').mockResolvedValue(mockTokenSupply as EsdtSupply); + }); + + it('should process only the tokens created in the meantime and include them in the result', async () => { + jest.spyOn(esdtService, 'getAllFungibleTokenProperties') + .mockResolvedValueOnce([new TokenProperties({ identifier: 'OLD-111111' })]) + .mockResolvedValueOnce([new TokenProperties({ identifier: 'OLD-111111' }), new TokenProperties({ identifier: 'NEW-222222' })]); + jest.spyOn(collectionService, 'getNftCollections') + .mockResolvedValueOnce([{ collection: 'OLDMETA-333333' } as NftCollection]) + .mockResolvedValueOnce([{ collection: 'OLDMETA-333333' } as NftCollection, { collection: 'NEWMETA-444444' } as NftCollection]); + + // snapshot the identifiers at call time, since the processed array is extended afterwards + const processedBatches: string[][] = []; + jest.spyOn(tokenService as any, 'batchProcessTokens').mockImplementation((tokens: any) => { + processedBatches.push(tokens.map((t: TokenDetailed) => t.identifier)); + return Promise.resolve(); + }); + + const result = await tokenService.getAllTokensRaw(); + + expect(processedBatches).toEqual([ + ['OLD-111111', 'OLDMETA-333333'], + ['NEW-222222', 'NEWMETA-444444'], + ]); + + expect(result.map(t => t.identifier).sort()).toEqual(['EGLD-000000', 'NEW-222222', 'NEWMETA-444444', 'OLD-111111', 'OLDMETA-333333']); + + const newToken = result.find(t => t.identifier === 'NEW-222222'); + expect(newToken?.type).toBe(TokenType.FungibleESDT); + expect(newToken?.supply).toBe(mockTokenSupply.totalSupply); + }); + + it('should keep the already processed tokens if fetching the latest tokens fails', async () => { + jest.spyOn(esdtService, 'getAllFungibleTokenProperties') + .mockResolvedValueOnce([new TokenProperties({ identifier: 'OLD-111111' })]) + .mockRejectedValueOnce(new Error('elastic unavailable')); + jest.spyOn(collectionService, 'getNftCollections').mockResolvedValue([{ collection: 'OLDMETA-333333' } as NftCollection]); + + const result = await tokenService.getAllTokensRaw(); + + expect((tokenService as any).batchProcessTokens).toHaveBeenCalledTimes(1); + expect(result.map(t => t.identifier).sort()).toEqual(['EGLD-000000', 'OLD-111111', 'OLDMETA-333333']); + }); + }); }); it('adjusts the order depending on the price source and market cap', async () => {