Thursday, July 9, 2020

Dynamodb : large data file upload to dynamodb


see this funciton saveIntoTableAtOnce


const express = require('express');
const request = require('request');
const router = express.Router(),
    moment = require('moment'),
    http = require('http'),
    fs = require('fs'),
    validator = require('validator'),
    validate = require('validate-vat'),
    validationEngine = require('../../custom_modules/validation_engine'),
    HashMap = require('hashmap'),
    stringHash = require('string-hash'),
    sort = require('alphanum-sort');




function saveIntoTableAtOnce(paramToSave, count, loopCount, declarationStatus, flowType, subFlowTypes, providerVatNumber, fiscalPeriod, savedData, flowWiseValueMapper, flowWiseFieldMapper, callback) {

console.log('loopCount: ' + loopCount + ' count: ' + count);

if (count >= loopCount) {

if (errorParamList.length > 0) {
var tempErrorParamList = errorParamList;
errorParamList = [];
console.log(tempErrorParamList);
loopCount = Math.ceil(tempErrorParamList.length / dataTosaveAtOnce);
saveIntoTableAtOnce(tempErrorParamList, 0, loopCount, declarationStatus, flowType, subFlowTypes, providerVatNumber, fiscalPeriod, savedData, flowWiseValueMapper, flowWiseFieldMapper, callback);
}
else
callback(null, true, declarationStatus, flowType, providerVatNumber, fiscalPeriod, subFlowTypes, savedData, flowWiseValueMapper, flowWiseFieldMapper);
}
var startPoint = count * dataTosaveAtOnce;
var endPoint = (count + 1) * dataTosaveAtOnce;
if (endPoint > paramToSave.length)
endPoint = paramToSave.length;
let dataToSaveAtThisInstance = endPoint - startPoint;
let innerCount = 0;

for (let i = startPoint; i < endPoint; i++) {

docClient.put(paramToSave[i], function (err, data) {
innerCount++;
if (err) {
if(err.message == "Requested resource not found"){
errorParamList.push(paramToSave[i]);
}
else{
console.log("paramToSave no." +i +" = "+JSON.stringify(paramToSave[i]));
console.error("Unable to add item. Error JSON:", JSON.stringify(err, null, 2));

let msg = JSON.stringify(err) +'\n';
let hashKey = stringHash( msg );
errorsMap.set( hashKey, msg );
}

}

console.log('INSERTED Data: ' + innerCount);

if (innerCount === dataToSaveAtThisInstance)
saveIntoTableAtOnce(paramToSave, count + 1, loopCount, declarationStatus, flowType, subFlowTypes, providerVatNumber, fiscalPeriod, savedData, flowWiseValueMapper, flowWiseFieldMapper, callback);

}); //end put
}
}

No comments:

Post a Comment

Autoboxing and Unboxing

  Autoboxing  is the automatic conversion that the Java compiler makes between the primitive types and their corresponding object wrapper cl...