Before we explore the use of generators for asynchronous control flow, it's important we learn some basic concepts. Let's start from the syntax; a generator function can be declared by appending the * (asterisk) operator after the function keyword:

function* makeGenerator() { 
  //body 
} 

Inside the makeGenerator() function, we can pause the execution using the keyword yield and return to the caller the value passed to it:

function* makeGenerator() { 
  yield 'Hello World'; 
  console.log('Re-entered'); 
} 

In the preceding code, the generator yields a string, Hello World, by putting the execution of the function on pause. When the generator is resumed, the execution will start from console.log('Re-entered').

The makeGenerator() function is essentially a factory that, when invoked, returns a new generator object:

const gen = makeGenerator(); 

The most important method of the generator object is next(), which is used to start/resume the execution of the generator and returns an object in the following form:

{ 
  value: <yielded value> 
  done: <true if the execution reached the end> 
} 

This object contains the value yielded by the generator (value) and a flag to indicate if the generator has completed its execution (done).

function* fruitGenerator() { 
    yield 'apple'; 
    yield 'orange'; 
    return 'watermelon'; 
} 
 
const newFruitGenerator = fruitGenerator(); 
console.log(newFruitGenerator.next());    //[1] 
console.log(newFruitGenerator.next());    //[2] 
console.log(newFruitGenerator.next());    //[3] 

{ value: 'apple', done: false }
{ value: 'orange', done: false }
{ value: 'watermelon', done: true }

function* iteratorGenerator(arr) { 
  for(let i = 0; i <arr.length; i++) { 
    yield arr[i]; 
  } 
} 
 
const iterator = iteratorGenerator(['apple', 'orange',  'watermelon']); 
let currentItem = iterator.next(); 
while(!currentItem.done) { 
  console.log(currentItem.value); 
  currentItem = iterator.next(); 
} 

apple
orange
watermelon

function* twoWayGenerator() { 
  const what = yield null; 
  console.log('Hello ' + what); 
} 
 
const twoWay = twoWayGenerator(); 
twoWay.next(); 
twoWay.next('world'); 

const twoWay = twoWayGenerator(); 
twoWay.next(); 
twoWay.throw(new Error()); 

You must be wondering how generators can help us with handling asynchronous operations. We can demonstrate this by creating a special function that accepts a generator as an argument and allows us to use asynchronous code inside the generator. This function takes care to resume the execution of the generator when the asynchronous operation completes. We will call this function asyncFlow():

function asyncFlow(generatorFunction) { 
  function callback(err) { 
    if(err) { 
      return generator.throw(err); 
    } 
    const results = [].slice.call(arguments, 1); 
    generator.next(results.length> 1 ? results : results[0]); 
  } 
  const generator = generatorFunction(callback); 
  generator.next(); 
} 

The preceding function takes a generator as input, instantiates it, and then immediately starts its execution:

const generator = generatorFunction(callback); 
generator.next(); 

The generatorFunction() receives as input, a special callback function that invokes generator.throw() if an error is received; otherwise, it resumes the execution of the generator by passing back the results received in the callback function:

if(err) { 
  return generator.throw(err); 
} 
const results = [].slice.call(arguments, 1); 
generator.next(results.length> 1 ? results : results[0]); 

To demonstrate the power of this simple function, let's create a new module called clone.js, which, for no meaningful reason, creates a clone of itself. Paste the asyncFlow() function we just created, followed by the core of the program:

const fs = require('fs'); 
const path = require('path'); 
 
asyncFlow(function* (callback) { 
  const fileName = path.basename(__filename); 
  const myself = yield fs.readFile(fileName, 'utf8', callback); 
  yield fs.writeFile(`clone_of_${filename}`, myself, callback); 
  console.log('Clone created'); 
}); 

Remarkably, with the help of the asyncFlow() function, we were able to write asynchronous code using a linear approach, as we were using blocking functions! The magic behind this result should be clear by now. The callback passed to each asynchronous function will in turn resume the generator as soon as the asynchronous operation is complete. Nothing complicated, but the outcome is surely impressive.

There are two other variations of this technique, one involving the use of promises and the other using thunks.

Both thunks and promises allow us to create generators that do not need a callback to be passed as an argument; for example, a version of asyncFlow() using thunks might be the following:

function asyncFlowWithThunks(generatorFunction) { 
  function callback(err) { 
    if(err) { 
      return generator.throw(err); 
    } 
    const results = [].slice.call(arguments, 1); 
    const thunk = generator.next(results.length> 1 ? results :  
                  results[0]).value; 
    thunk && thunk(callback); 
  } 
  const generator = generatorFunction(); 
  const thunk = generator.next().value; 
  thunk && thunk(callback); 
} 

The trick is to read the return value of generator.next(), which contains the thunk. The next step is to invoke the thunk itself by injecting our special callback. Simple! This allows us to write the following code:

asyncFlowWithThunks(function* () { 
  const fileName = path.basename(__filename); 
  const myself = yield readFileThunk(__filename, 'utf8'); 
  yield writeFileThunk(`clone_of_${fileName}`, myself); 
  console.log("Clone created"); 
}); 

In the same way, we could implement a version of asyncFlow() that accepts a promise as yieldable. We leave this as an exercise as its implementation requires only a minimal change to the asyncFlowWithThunks() function. We may also implement an asyncFlow() function that accepts both promises and thunks as yieldables, using the same principles.

Let's start our practical exploration of generators and co by modifying version 2 of the web spider application. The very first thing we have to do is to load our dependencies and generate a thunkified version of the functions we are going to use. These will go at the top of the spider.js module:

const thunkify = require('thunkify'); 
const co = require('co'); 
 
const request = thunkify(require('request')); 
const fs = require('fs'); 
const mkdirp = thunkify(require('mkdirp')); 
const readFile = thunkify(fs.readFile); 
const writeFile = thunkify(fs.writeFile); 
const nextTick = thunkify(process.nextTick); 

Looking at the preceding code, we can surely notice some similarities with the code we used earlier in the chapter to promisify some APIs. In this regard, it is interesting to point out that if we decided to use the promisified version of our functions instead of their thunkified alternative, the code would remain exactly the same, thanks to the fact that co supports both thunks and promises as yieldable objects. In fact, if we want, we could even use both thunks and promises in the same application, even in the same generator. This is a tremendous advantage in terms of flexibility, as it allows us to use a generator-based control flow with whatever solution we already have at our disposal.

Okay, now let's start transforming the download() function into a generator:

function* download(url, filename) { 
  console.log(`Downloading ${url}`); 
  const response = yield request(url); 
  const body = response[1]; 
  yield mkdirp(path.dirname(filename)); 
  yield writeFile(filename, body); 
  console.log(`Downloaded and saved ${url}`); 
  return body; 
} 

By using generators and co, our download() function suddenly becomes trivial. All we had to do is convert it into a generator function and use yield wherever we had an asynchronous function (as thunk) to invoke.

Next, it's the turn of the spider() function:

function* spider(url, nesting) { 
  const filename = utilities.urlToFilename(url); 
  let body; 
  try { 
    body = yield readFile(filename, 'utf8'); 
  } catch(err) { 
    if(err.code !== 'ENOENT') { 
      throw err; 
    } 
    body = yield download(url, filename); 
  } 
  yield spiderLinks(url, body, nesting); 
} 

The interesting detail to notice from this last fragment of code is how we were able to use a try...catch block to handle exceptions. Also, we can now use throw to propagate errors! Another remarkable line is where we yield the download() function, which is not a thunk nor a promisified function, but just another generator. This is possible, thanks to co, which also supports other generators as yieldables.

Finally, we can also convert spiderLinks(), where we implemented an iteration to download the links of a web page in sequence. With generators, this becomes trivial as well:

function* spiderLinks(currentUrl, body, nesting) { 
  if(nesting === 0) { 
    return nextTick(); 
  } 
 
  const links = utilities.getPageLinks(currentUrl, body); 
  for(let i = 0; i <links.length; i++) { 
    yield spider(links[i], nesting - 1); 
  } 
} 

There is little to explain from the previous code. There is no pattern to show for the sequential iteration; generators and co are doing all the dirty work for us, so we were able to write the asynchronous iteration as if we were using blocking, direct style APIs.

Now comes the most important part, the entry point of our program:

co(function* () { 
  try { 
    yield spider(process.argv[2], 1); 
    console.log('Download complete'); 
  } catch(err) { 
    console.log(err); 
  } 
}); 

This is the only place where we have to invoke co(...) to wrap a generator. In fact, once we do that, co will automatically wrap any generator we pass to a yield statement, and this will happen recursively, so the rest of the program is totally agnostic to the fact we are using co, even though it's under the hood.

Now it should be possible to run our generator-based web spider application.

The bad news about generators is that they are great for writing sequential algorithms, but they can't be used to parallelize the execution of a set of tasks, at least not just using yield and generators. In fact, the pattern to use in these circumstances is to simply rely on a callback-based or promise-based function, which in turn can easily be yielded and used with generators.

Luckily, for the specific case of the unlimited parallel execution, co already allows us to obtain it natively by simply yielding an array of promises, thunks, generators, or generator functions.

With this in mind, version 3 of our web spider application can be implemented simply by rewriting the spiderLinks() function as follows:

function* spiderLinks(currentUrl, body, nesting) { 
  if(nesting === 0) { 
    return nextTick(); 
  } 
 
  const links = utilities.getPageLinks(currentUrl, body); 
  const tasks = links.map(link => spider(link, nesting - 1)); 
  yield tasks; 
} 

What we did was just to collect all the download tasks, which are essentially generators, and then yield on the resulting array. All these tasks will be executed by co in parallel and then the execution of our generator (spiderLinks) will be resumed when all the tasks finish running.

If you think we cheated by exploiting the feature of co that allows us to yield on an array, it is possible to demonstrate how the same parallel flow can be achieved using a callback-based solution similar to what we have already used earlier in the chapter. Let's use this technique to rewrite spiderLinks() once again:

function spiderLinks(currentUrl, body, nesting) { 
  if(nesting === 0) { 
    return nextTick(); 
  } 
 
  //returns a thunk 
  return callback => { 
    let completed = 0, hasErrors = false; 
    const links = utilities.getPageLinks(currentUrl, body); 
    if(links.length === 0) { 
      return process.nextTick(callback); 
    } 
 
    function done(err, result) { 
      if(err && !hasErrors) { 
        hasErrors = true; 
        return callback(err); 
      } 
      if(++completed === links.length && !hasErrors) { 
        callback(); 
      } 
    } 
 
    for(let i = 0; i < links.length; i++) { 
      co(spider(links[i], nesting - 1)).then(done); 
    } 
  } 
} 

To run the spider() function in parallel, we use co, which executes the generator and returns a promise. This way, we are able to wait for the promise to be resolved and call the done() function. Usually, all the libraries for generator-based control flows have similar features, so you can always transform a generator into a callback-based or a promise-based function if needed.

To start multiple download tasks in parallel, we just reused the callback-based pattern for parallel execution defined earlier in the chapter. We should also notice that we transformed the spiderLinks() function to a thunk (it's not even a generator anymore.) This enabled us to have a callback function to invoke when all the parallel tasks are completed.

Now that we know what to do with nonsequential execution flows, it should be easy to plan the implementation of version 4 of our web spider application, the one imposing a limit on the number of concurrent download tasks. We have several options we can use to do that; some of them are as follows:

For educational purposes, we are going to choose the last option, so we can dive into a pattern that is often associated with coroutines (but also threads and processes).

class TaskQueue { 
  constructor(concurrency) { 
    this.concurrency = concurrency; 
    this.running = 0; 
    this.taskQueue = []; 
    this.consumerQueue = []; 
    this.spawnWorkers(concurrency); 
  } 
 
  pushTask(task) { 
    if (this.consumerQueue.length !== 0) { 
      this.consumerQueue.shift()(null, task); 
    } else { 
      this.taskQueue.push(task); 
    } 
  } 
 
  spawnWorkers(concurrency) { 
    const self = this; 
    for(let i = 0; i < concurrency; i++) { 
      co(function* () { 
        while(true) { 
          const task = yield self.nextTask(); 
          yield task; 
        } 
      }); 
    } 
  } 
 
  nextTask() { 
    return callback => { 
      if(this.taskQueue.length !== 0) { 
        return callback(null, this.taskQueue.shift()); 
      } 
 
      this.consumerQueue.push(callback); 
    } 
  } 
} 

nextTask() { 
  return callback => { 
    if(this.taskQueue.length !== 0) { 
      return callback(null, this.taskQueue.shift()); 
    } 
    this.consumerQueue.push(callback); 
  } 
} 

const TaskQueue = require('./taskQueue'); 
const downloadQueue = new TaskQueue(2); 

function spiderLinks(currentUrl, body, nesting) { 
  //... 
  return (callback) => { 
    //... 
    function done(err, result) { 
      //... 
    } 
    links.forEach(function(link) { 
      downloadQueue.pushTask(function *() { 
        yield spider(link, nesting - 1); 
        done(); 
      }); 
    }); 
  } 
}