add -C option to hlsrecord to collect segments into a single file

bin/hlsrecord

@@ -5,67 +5,72 @@
 
 // record a live hls-stream storing an on-demand ready version
 
-var hlsrecord = require('commander');
+var fs = require('fs'),
+    path = require('path');
+var nopt = require('noptify/node_modules/nopt');
+
+var HlsSegmentReader = require('hls-segment-reader');
+var recorder = require('../lib/recorder');
+
+function DateValue(){}
+nopt.typeDefs[DateValue] = { type: DateValue, validate: function (data, key, val) {
+  var date;
+  if (val === 'now') val = '+0';
+  if (val.length && (val[0] === '+' || val[0] === '-')) {
+    date = new Date(Math.round(new Date().getTime() / 1000 + parseInt(val, 10)) * 1000);
+  } else if (parseInt(val, 10) == val) {
+    date = new Date(parseInt(val, 10) * 1000);
+  } else {
+    data = new Date(val);
+  }
+  if (!date) return false;
+  data[key] = date;
+}};
+
+var hlsrecord = require('noptify')(process.argv, { program: 'hlsrecord <url>' });
 hlsrecord.version(require('../package').version)
-   .usage('[options] <url>')
-   .option('-o, --output <dir>', 'Output directory')
-   .option('-c, --create-dir', 'Explicitly create output dir')
-   .option('-b, --begin-date <date>', 'Start recording at', dateValue)
-   .option('-e, --end-date <date>', 'Stop recording at', dateValue)
-   .option('-s, --start-offset <seconds>', 'Playback start time offset', parseFloat)
-   .option('--extension <label>', 'preserve vendor extension', function (val) {
-     return (hlsrecord.extension || []).concat(val);
-   })
-   .option('--segment-ext <label>', 'preserve vendor segment extension', function (val) {
-     return (hlsrecord.segmentExt || []).concat(val);
-   })
+   .option('collect', '-C', 'Collect output segments to a single file', Boolean)
+   .option('output', '-o', 'Output directory', path)
+   .option('create-dir', '-c', 'Explicitly create output dir', Boolean)
+   .option('begin-date', '-b', 'Start recording at', DateValue)
+   .option('end-date', '-e', 'Stop recording at', DateValue)
+   .option('start-offset', '-s', 'Playback start time offset in seconds', Number)
+   .option('create-dir', '-c', 'Explicitly create output dir', Boolean)
+   .option('extension', 'Preserve specified vendor extension', Array)
+   .option('segment-extension', 'Preserve specified vendor segment extension', Array)
 //   .option('-a, --user-agent <string>', 'HTTP User-Agent')
 //   .option('-f, --full', 'record all variants')
    .parse(process.argv);
 
-function dateValue(val) {
-  // FIXME: negative values doesn't work with commander - https://github.com/visionmedia/commander.js/issues/61
-  if (val === 'now') return new Date();
-  if (val.length && (val[0] === '+' || val[0] === '-'))
-    return new Date(Math.round(new Date().getTime() / 1000 + parseInt(val, 10)) * 1000);
-  if (parseInt(val, 10) == val)
-    return new Date(parseInt(val, 10) * 1000);
-  return new Date(val);
-}
-
-var fs = require('fs');
-
-var HlsSegmentReader = require('hls-segment-reader');
-var recorder = require('../lib/recorder');
-
-var src = hlsrecord.args[0];
+var options = hlsrecord.nopt;
+var src = options.argv.remain[0];
 if (!src) {
   hlsrecord.help();
   process.exit(-1);
 }
 
-var outDir = hlsrecord.output || 'stream';
-if (hlsrecord.createDir)
+var outDir = options.output || 'stream';
+if (options['create-dir'])
   fs.mkdirSync(outDir);
 
-if (hlsrecord.beginDate)
-  console.log('fetching from:', hlsrecord.beginDate);
-if (hlsrecord.endDate)
-  console.log('fetching until:', hlsrecord.endDate);
+if (options['begin-date'])
+  console.log('fetching from:', options['begin-date']);
+if (options['end-date'])
+  console.log('fetching until:', options['end-date']);
 
 var extensions = {};
-(hlsrecord.extension || []).forEach(function(ext) {
+(options.extension || []).forEach(function(ext) {
   extensions[ext] = false;
 });
-(hlsrecord.segmentExt || []).forEach(function(ext) {
+(options['segment-extension'] || []).forEach(function(ext) {
   extensions[ext] = true;
 });
 
 var readerOptions = {
   withData: true,
-  fullStream: !hlsrecord.beginDate,
-  startDate: hlsrecord.beginDate,
-  stopDate: hlsrecord.endDate,
+  fullStream: !options['begin-date'],
+  startDate: options['begin-date'],
+  stopDate: options['end-date'],
   maxStallTime: 5 * 60 * 1000,
   extensions: extensions,
   highWaterMark: 0,
@@ -80,4 +85,4 @@ function createReader(src) {
 }
 
 var rdr = createReader(src);
-recorder(rdr, outDir, { subreader:createReader, startOffset: hlsrecord.startOffset }).start();
+recorder(rdr, outDir, { subreader:createReader, startOffset: options['start-offset'], collect: options.collect }).start();

lib/recorder.js

@@ -31,6 +31,7 @@ function HlsStreamRecorder(reader, dst, options) {
 
   this.startOffset = parseFloat(options.startOffset);
   this.subreader = options.subreader;
+  this.collect = !!options.collect; // collect into a single file (v4 feature)
 
   this.recorders = [];
 }
@@ -54,20 +55,22 @@ HlsStreamRecorder.prototype.updateIndex = function(update) {
   if (!this.index) {
     this.index = new m3u8parse.M3U8Playlist(update);
     if (!this.index.variant) {
-      if (this.index.version < 2) {
-        // version 2 is required to support the remapped IV attribute
-        this.index.version = 2;
+      if (this.collect)
+        this.index.version = Math.max(4, this.index.version);  // v4 is required for byterange support
+      this.index.version = Math.max(2, this.index.version);    // v2 is required to support the remapped IV attribute
+      if (this.index.version !== update.version)
         debug('changed index version to:', this.index.version);
-      }
       this.index.segments = [];
-      this.index.first_seq_no = self.seq;
+      this.index.first_seq_no = this.seq;
       this.index.type = 'EVENT';
       this.index.ended = false;
       this.index.discontinuity_sequence = 0; // not allowed in event playlists
       if (!isNaN(this.startOffset)) {
         var offset = this.startOffset;
-        if (offset < 0) offset = Math.min(offset, -3 * this.target_duration);
-        this.index.start.decimalInteger('offset', offset);
+        if (!update.ended) {
+          if (offset < 0) offset = Math.min(offset, -3 * this.index.target_duration);
+        }
+        this.index.start.decimalInteger('time-offset', offset);
       }
     } else {
       debug('programs', this.index.programs);
@@ -162,9 +165,6 @@ HlsStreamRecorder.prototype.process = function(segmentInfo, next) {
   var segment = new m3u8parse.M3U8Segment(segmentInfo.details, true);
   var meta = segmentInfo.file;
 
-  // remove byterange info
-  delete segment.byterange;
-
   // mark discontinuities
   if (this.nextSegmentSeq !== -1 &&
       this.nextSegmentSeq !== segmentInfo.seq)
@@ -174,13 +174,36 @@ HlsStreamRecorder.prototype.process = function(segmentInfo, next) {
   // create our own uri
   segment.uri = util.format('%s.%s', this.segmentName(this.seq), mime.extension(meta.mime));
 
+  // handle byterange
+  var first = self.index.segments.length === 0;
+  var newFile = first || self.index.segments[self.index.segments.length - 1].uri !== segment.uri;
+  if (this.collect) {
+    segment.byterange = {
+      length: 0,
+      offset: newFile ? 0 : null
+    }
+  } else {
+    delete segment.byterange;
+  }
+
   // save the stream segment
   var stream = oncemore(segmentInfo.stream);
-  stream.pipe(fs.createWriteStream(path.join(this.dst, segment.uri)));
+  stream.pipe(fs.createWriteStream(path.join(this.dst, segment.uri), { flags: newFile ? 'w' : 'a' }));
+
+  var bytesWritten = 0;
+  if (this.collect) {
+    stream.on('data', function(chunk) {
+      bytesWritten += chunk.length;
+    });
+  }
+
   stream.once('end', 'error', function(err) {
     // only to report errors
     if (err) debug('stream error', err.stack || err);
 
+    if (segment.byterange)
+      segment.byterange.length = bytesWritten;
+
     // update index
     self.index.segments.push(segment);
     self.flushIndex(next);
@@ -206,7 +229,7 @@ HlsStreamRecorder.prototype.segmentName = function(seqNo) {
     if (next) return name(next - 1) + chr;
     return chr;
   }
-  return name(seqNo);
+  return this.collect ? 'stream' : name(seqNo);
 };
 
 HlsStreamRecorder.prototype.flushIndex = function(cb) {

package.json

@@ -34,6 +34,7 @@
     "measured": "^1.0.0",
     "mime-types": "^2.0.1",
     "mkdirp": "^0.5.0",
+    "noptify": "0.0.3",
     "oncemore": "^1.0.0",
     "readable-stream": "~1.0.0",
     "streamprocess": "^1.0.0",