Fixes/gcs fixes (#2438)

* Ensure only files are uploaded since the node lib does not support uloading folders directly

* Addig flag for enabeling gzip metadata encoding and disabled resumable upload since to many connecteins are opened some times.

* Running the linter to late as usual
This commit is contained in:
Markus Liljedahl 2019-04-26 12:54:17 +02:00 committed by Peter Hedenskog
parent 247074e777
commit 6b3209e4b3
2 changed files with 20 additions and 3 deletions

View File

@ -917,6 +917,13 @@ module.exports.parseCommandLine = function parseCommandLine() {
type: 'boolean',
group: 'GoogleCloudStorage'
})
.option('gcs.gzip', {
describe:
'Add content-encoding for gzip to the uploaded files. Read more at https://cloud.google.com/storage/docs/transcoding. If you host your results directly from the bucket, gzip must be set to false',
default: false,
type: 'boolean',
group: 'GoogleCloudStorage'
})
.option('gcs.path', {
describe:
"Override the default folder path in the bucket where the results are uploaded. By default it's " +

View File

@ -21,7 +21,13 @@ async function upload(dir, gcsOptions, prefix) {
const bucket = storage.bucket(gcsOptions.bucketname);
for (let file of files) {
promises.push(uploadFile(file, bucket, gcsOptions, prefix, dir));
const stats = fs.statSync(file);
if (stats.isFile()) {
promises.push(uploadFile(file, bucket, gcsOptions, prefix, dir));
} else {
log.debug(`Will not upload ${file} since it is not a file`);
}
}
return Promise.all(promises);
}
@ -33,9 +39,9 @@ async function uploadFile(file, bucket, gcsOptions, prefix, baseDir) {
const params = {
public: !!gcsOptions.public,
destination: fileName,
resumable: true,
resumable: false,
validation: 'crc32c',
gzip: true,
gzip: !!gcsOptions.gzip,
metadata: {
metadata: {
cacheControl: 'public, max-age=31536000'
@ -83,6 +89,10 @@ module.exports = {
if (gcsOptions.removeLocalResult) {
await fs.remove(baseDir);
log.debug(`Removed local files and directory ${baseDir}`);
} else {
log.debug(
`Local result files and directories are stored in ${baseDir}`
);
}
} catch (e) {
log.error('Could not upload to Google Cloud Storage', e);