From 713798cd58fee0050bf2de876abffd4096416a8a Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Fri, 11 Sep 2026 12:38:17 +0300 Subject: [PATCH 1/4] check tokens created within all tokens processing time window --- src/endpoints/tokens/token.service.ts | 54 +++++++++++++++++++++------ 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/src/endpoints/tokens/token.service.ts b/src/endpoints/tokens/token.service.ts index b9a3b9f41..a9bc2bd77 100644 --- a/src/endpoints/tokens/token.service.ts +++ b/src/endpoints/tokens/token.service.ts @@ -761,9 +761,30 @@ export class TokenService { } this.logger.log(`Starting to fetch all tokens`); + let tokens = await this.fetchAllTokensWithoutDetails(); + + await this.applyTokenDetails(tokens); + + await this.applyTokensCreatedDuringProcessing(tokens); + + this.logger.log(`Sorting and finalizing ${tokens.length} tokens`); + tokens = tokens.sortedDescending( + token => token.assets ? 1 : 0, + token => token.marketCap ? 1 : 0, + token => token.isLowLiquidity || token.assets?.priceSource?.type === TokenAssetsPriceSourceType.customUrl ? 0 : (token.marketCap ?? 0), + token => token.transactions ?? 0, + ); + + tokens = [...tokens, await this.buildEgldToken()]; + + this.logger.log(`Total tokens processed: ${tokens.length}`); + return tokens; + } + + private async fetchAllTokensWithoutDetails(): Promise { const startFungible = Date.now(); const tokensProperties = await this.esdtService.getAllFungibleTokenProperties(); - let tokens = tokensProperties.map(properties => ApiUtils.mergeObjects(new TokenDetailed(), properties)); + const tokens = tokensProperties.map(properties => ApiUtils.mergeObjects(new TokenDetailed(), properties)); this.logger.log(`Fetched ${tokens.length} fungible tokens in ${Date.now() - startFungible}ms`); const allAssets = await this.assetsService.getAllTokenAssets(); @@ -785,20 +806,29 @@ export class TokenService { tokens.push(this.buildMetaEsdtToken(collection)); } - await this.applyTokenDetails(tokens); + return tokens; + } - this.logger.log(`Sorting and finalizing ${tokens.length} tokens`); - tokens = tokens.sortedDescending( - token => token.assets ? 1 : 0, - token => token.marketCap ? 1 : 0, - token => token.isLowLiquidity || token.assets?.priceSource?.type === TokenAssetsPriceSourceType.customUrl ? 0 : (token.marketCap ?? 0), - token => token.transactions ?? 0, - ); + private async applyTokensCreatedDuringProcessing(tokens: TokenDetailed[]): Promise { + // processing all tokens takes tens of seconds, so re-fetch the token list and + // process only the tokens created in the meantime, instead of waiting for the next refresh + try { + const processedIdentifiers = new Set(tokens.map(token => token.identifier)); + const latestTokens = await this.fetchAllTokensWithoutDetails(); + const newTokens = latestTokens.filter(token => !processedIdentifiers.has(token.identifier)); - tokens = [...tokens, await this.buildEgldToken()]; + if (newTokens.length === 0) { + return; + } - this.logger.log(`Total tokens processed: ${tokens.length}`); - return tokens; + this.logger.log(`Processing ${newTokens.length} tokens created while processing all tokens`); + await this.applyTokenDetails(newTokens); + + tokens.push(...newTokens); + } catch (error) { + this.logger.error('Could not apply tokens created while processing all tokens'); + this.logger.error(error); + } } async getTokenRaw(rawIdentifier: string): Promise { From f6ea144fd81d0482400cdae8abaae958983c029f Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Fri, 11 Sep 2026 12:41:27 +0300 Subject: [PATCH 2/4] unit tests --- src/test/unit/services/tokens.spec.ts | 64 ++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/src/test/unit/services/tokens.spec.ts b/src/test/unit/services/tokens.spec.ts index 6e7ea9488..fb4ebedc2 100644 --- a/src/test/unit/services/tokens.spec.ts +++ b/src/test/unit/services/tokens.spec.ts @@ -723,7 +723,8 @@ describe('Token Service', () => { expect(apiConfigService.isTokensFetchFeatureEnabled).toHaveBeenCalled(); expect(esdtService.getAllFungibleTokenProperties).toHaveBeenCalled(); - expect(assetsService.getAllTokenAssets).toHaveBeenCalledTimes(1); + // the token list is fetched once for processing and once more to catch tokens created in the meantime + expect(assetsService.getAllTokenAssets).toHaveBeenCalledTimes(2); mockTokens.forEach(mockToken => { mockToken.name = mockTokenAssets.name; @@ -797,6 +798,67 @@ describe('Token Service', () => { expect(egldToken?.supply).toBe('0'); expect(egldToken?.circulatingSupply).toBe('0'); }); + + describe('tokens created while processing all tokens', () => { + const mockTokenSupply: Partial = { totalSupply: '1000', circulatingSupply: '1000' }; + + beforeEach(() => { + jest.spyOn(apiConfigService, 'isTokensFetchFeatureEnabled').mockReturnValue(false); + jest.spyOn(assetsService, 'getAllTokenAssets').mockResolvedValue({}); + jest.spyOn(assetsService, 'getTokenAssets').mockResolvedValue(undefined); + + jest.spyOn(tokenService as any, 'batchProcessTokens').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexLiquidity').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexPrices').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexPairType').mockImplementation(() => Promise.resolve()); + jest.spyOn(tokenService as any, 'applyMexPairTradesCount').mockImplementation(() => Promise.resolve()); + jest.spyOn(cacheService as any, 'batchApplyAll').mockImplementation(() => Promise.resolve()); + jest.spyOn(dataApiService, 'getEsdtTokenPrice').mockResolvedValue(undefined); + jest.spyOn(dataApiService, 'getEgldPrice').mockResolvedValue(100); + jest.spyOn(esdtService, 'getTokenSupply').mockResolvedValue(mockTokenSupply as EsdtSupply); + }); + + it('should process only the tokens created in the meantime and include them in the result', async () => { + jest.spyOn(esdtService, 'getAllFungibleTokenProperties') + .mockResolvedValueOnce([new TokenProperties({ identifier: 'OLD-111111' })]) + .mockResolvedValueOnce([new TokenProperties({ identifier: 'OLD-111111' }), new TokenProperties({ identifier: 'NEW-222222' })]); + jest.spyOn(collectionService, 'getNftCollections') + .mockResolvedValueOnce([{ collection: 'OLDMETA-333333' } as NftCollection]) + .mockResolvedValueOnce([{ collection: 'OLDMETA-333333' } as NftCollection, { collection: 'NEWMETA-444444' } as NftCollection]); + + // snapshot the identifiers at call time, since the processed array is extended afterwards + const processedBatches: string[][] = []; + jest.spyOn(tokenService as any, 'batchProcessTokens').mockImplementation((tokens: any) => { + processedBatches.push(tokens.map((t: TokenDetailed) => t.identifier)); + return Promise.resolve(); + }); + + const result = await tokenService.getAllTokensRaw(); + + expect(processedBatches).toEqual([ + ['OLD-111111', 'OLDMETA-333333'], + ['NEW-222222', 'NEWMETA-444444'], + ]); + + expect(result.map(t => t.identifier).sort()).toEqual(['EGLD-000000', 'NEW-222222', 'NEWMETA-444444', 'OLD-111111', 'OLDMETA-333333']); + + const newToken = result.find(t => t.identifier === 'NEW-222222'); + expect(newToken?.type).toBe(TokenType.FungibleESDT); + expect(newToken?.supply).toBe(mockTokenSupply.totalSupply); + }); + + it('should keep the already processed tokens if fetching the latest tokens fails', async () => { + jest.spyOn(esdtService, 'getAllFungibleTokenProperties') + .mockResolvedValueOnce([new TokenProperties({ identifier: 'OLD-111111' })]) + .mockRejectedValueOnce(new Error('elastic unavailable')); + jest.spyOn(collectionService, 'getNftCollections').mockResolvedValue([{ collection: 'OLDMETA-333333' } as NftCollection]); + + const result = await tokenService.getAllTokensRaw(); + + expect((tokenService as any).batchProcessTokens).toHaveBeenCalledTimes(1); + expect(result.map(t => t.identifier).sort()).toEqual(['EGLD-000000', 'OLD-111111', 'OLDMETA-333333']); + }); + }); }); it('adjusts the order depending on the price source and market cap', async () => { From bd358e99a9d3fa376795e218001d8b1936a2113e Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Fri, 11 Sep 2026 12:49:00 +0300 Subject: [PATCH 3/4] better logging --- src/endpoints/tokens/token.service.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/endpoints/tokens/token.service.ts b/src/endpoints/tokens/token.service.ts index a9bc2bd77..690d167ad 100644 --- a/src/endpoints/tokens/token.service.ts +++ b/src/endpoints/tokens/token.service.ts @@ -813,6 +813,7 @@ export class TokenService { // processing all tokens takes tens of seconds, so re-fetch the token list and // process only the tokens created in the meantime, instead of waiting for the next refresh try { + const startFetch = Date.now(); const processedIdentifiers = new Set(tokens.map(token => token.identifier)); const latestTokens = await this.fetchAllTokensWithoutDetails(); const newTokens = latestTokens.filter(token => !processedIdentifiers.has(token.identifier)); @@ -821,10 +822,13 @@ export class TokenService { return; } - this.logger.log(`Processing ${newTokens.length} tokens created while processing all tokens`); + const startProcessing = Date.now(); await this.applyTokenDetails(newTokens); tokens.push(...newTokens); + + const endProcessing = Date.now(); + this.logger.log(`Processed ${newTokens.length} tokens created while processing all tokens in ${endProcessing - startProcessing}ms (${endProcessing - startFetch}ms including re-fetch)`); } catch (error) { this.logger.error('Could not apply tokens created while processing all tokens'); this.logger.error(error); From 27fbe52832213395d20d313467febd5c765d4774 Mon Sep 17 00:00:00 2001 From: GuticaStefan Date: Fri, 11 Sep 2026 12:52:24 +0300 Subject: [PATCH 4/4] logging improvements --- src/endpoints/tokens/token.service.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/endpoints/tokens/token.service.ts b/src/endpoints/tokens/token.service.ts index 690d167ad..4ec0f8349 100644 --- a/src/endpoints/tokens/token.service.ts +++ b/src/endpoints/tokens/token.service.ts @@ -813,7 +813,7 @@ export class TokenService { // processing all tokens takes tens of seconds, so re-fetch the token list and // process only the tokens created in the meantime, instead of waiting for the next refresh try { - const startFetch = Date.now(); + const startFetchAndProcess = Date.now(); const processedIdentifiers = new Set(tokens.map(token => token.identifier)); const latestTokens = await this.fetchAllTokensWithoutDetails(); const newTokens = latestTokens.filter(token => !processedIdentifiers.has(token.identifier)); @@ -822,13 +822,12 @@ export class TokenService { return; } - const startProcessing = Date.now(); await this.applyTokenDetails(newTokens); tokens.push(...newTokens); - const endProcessing = Date.now(); - this.logger.log(`Processed ${newTokens.length} tokens created while processing all tokens in ${endProcessing - startProcessing}ms (${endProcessing - startFetch}ms including re-fetch)`); + const endFetchAndProcess = Date.now(); + this.logger.log(`Processed ${newTokens.length} tokens created while processing all tokens in ${endFetchAndProcess - startFetchAndProcess}ms`); } catch (error) { this.logger.error('Could not apply tokens created while processing all tokens'); this.logger.error(error);