Our current application architecture has coupled together an API; a consuming web client and a worker which populates a Redis cache. This approach works for many applications and will allow it to scale horizontally with the help of a load balancer.
But let's say for example, we would like our API to support clients other than web, say for example, we introduced a mobile client that used our API; ideally we would like to scale our API in isolation and remove anything related to the web client.
Scaling our worker horizontally would simply mean replicating the same work over and over again, which would be pointless. Later, we will discuss how to scale the worker.
In the rest of this chapter we will outline how to split apart our application in order to scale horizontally. We will use the source code from the chapter-6 version of the vision application. We will, of course, document anything of interest which is required to achieve our goal. We will create four new projects: vision-core, vision-web, vision-api, and vision-worker.
Tip
Download the source code for this chapter here:
https://github.com/AndrewKeig/vision-core
https://github.com/AndrewKeig/vision-web
Our first task is to extract everything that can be shared between the vision-web, vision-api, and vision-worker projects into a new vision-core project.
This includes the following sections: ./cache, ./lib/configuration, ./lib/db, ./lib/github, ./lib/logger, ./lib/models, and ./lib/project.
The vision-core project is not an application so we remove everything in the root of the project, including ./app.js and our ./gruntfile.js, and add a ./index.js file, which simply exports all of the functionalities shown:
module.exports.redis = require('./lib/cache/redis'),
module.exports.publisher = require('./lib/cache/publisher'),
module.exports.subscriber = require('./lib/cache/subscriber'),
module.exports.configuration = require('./lib/configuration'),
module.exports.db = require('./lib/db'),
module.exports.github = require('./lib/github'),
module.exports.project = require('./lib/project'),
module.exports.logger = require('./lib/logger'),
module.exports.models = require('./lib/models'),In order to share the private vision-core project with visions other private projects, we add a GitHub dependency to config: ./config/packge.json:
"dependencies": {
"vision-core": "git+ssh://[email protected]:AndrewKeig/vision-core.git#master",Let's create a vision-api project which contains the web API. Here we need to reuse everything related to the API that includes the following middleware: ./lib/middleware/id, ./lib/middleware/notFound, the routes for ./lib/routes/project, ./lib/routes/github, and ./lib/routes/heartbeat. We also include the config files ./config and all the tests ./test.
In order to secure vision-api, we will use basic authentication, which uses a username and password to authenticate a user. These credentials are transported in plain text, so you are advised to use HTTPS. We have already shown you how to setup HTTPS, hence, this part will not be repeated. In order to set up basic authentication, we can use the passport-http; let's install it:
npm install passport-http ––save
We start by adding a username and password to ./config/*.json:
"api": {
"username": "airasoul",
"password": "1234567890"
}We are now ready to implement an ApiAuth strategy into ./lib/auth/index.js. We start by defining a function, ApiAuth, then we import the passport and passport-http modules. We instantiate a BasicStrategy function and add it to passport, passing a verify function. Inside this verify function, we have the option of rejecting the user by passing false out of the callback. We call findUser and check if username and password are the same as those stored in ./config/*.json.
var config = require('vision-core').configuration;
function ApiAuth() {
this.passport = require('passport'),
var BasicStrategy = require('passport-http').BasicStrategy;
this.passport.use(new BasicStrategy({
},
function(username, password, done) {
findUser(username, password, function(err, status) {
return done(null, status);
})
}
));
var findUser = function(username, password, callback){
var usernameOk = config.get('api:username') === username;
var passwordOk = config.get('api:password') === password;
callback(null, usernameOk === passwordOk);
}
};
module.exports = new ApiAuth();The vision-api project will need a new Express server ./express/index.js. We start by requiring config via vision-core. We require the apiAuth module which handles authentication, then we apply the passport basic middleware to all of the routes using app.all. We set session:false as basic authentication is stateless.
var express = require('express')
, http = require('http')
, config = require('vision-core').configuration
, db = require('vision-core').db
, apiAuth = require('../auth')
, middleware = require('../middleware')
, routes = require('../routes')
, app = express();
app.set('port', config.get('express:port'));
app.use(express.logger({ immediate: true, format: 'dev' }));
app.use(express.bodyParser());
app.use(apiAuth.passport.initialize());
app.use(app.router);
app.all('*', apiAuth.passport.
authenticate('basic', { session: false }));
app.param('id', middleware.id.validate);
app.get('/heartbeat', routes.heartbeat.index);
app.get('/project/:id', routes.project.get);
app.get('/project', routes.project.all);
app.post('/project', routes.project.post);
app.put('/project/:id', routes.project.put);
app.del('/project/:id', routes.project.del);
app.get('/project/:id/repos', routes.github.repos);
app.get('/project/:id/commits', routes.github.commits);
app.get('/project/:id/issues', routes.github.issues);
app.use(middleware.notFound.index);
http.createServer(app).listen(app.get('port'));
module.exports = app;As we are moving to multiple Express servers to support our application, we will move vision-api onto port 3001. Let's configure this into ./config/*.json, as shown in the following code:
"express": {
"port": 3001
}Let's continue and create a new project called vision-worker, which consists of two scripts ./populate.js script and ./lib/cache/populate.js.
Of course we could scale this worker with something such as RabbitMQ. This would allow us to spawn multiple producers and consumers, and from this respect, the solution we have is not optimum. If you are interested in improving this part of the application, please refer to Packt's Instant RabbitMQ Message Application Development. This book explains how you can implement a worker pattern with RabbitMQ.
Finally, we create a new project called vision-web which will include everything related to the web client ; simply include everything from chapter 6 and remove everything we moved to core and reference core from ./package.json. Our current set of routes require some significant changes; now that we have decoupled our service layer into its own repository called vision-api. vision-web will no longer make service calls directly into the project and github services; these services now exist in the vision-api project, instead we will call the API services exposed on vision-api.
Let's add the configuration to ./config/*.json for our vision-api project. The vision-api project has been configured to run on port 3001 and uses basic authentication for security, so we include the username and password in the url.
"api": {
"url": "http://airasoul:[email protected]:3001"
}In order to call services on our vision-api project , we will simplify things by using Request module. Request is a simple client that allows us to make HTTP requests; lets install it:
npm install request --save
With our configuration in place, we move onto our project route ./lib/routes/project.js. Here we simply replace all calls to our Project service with the corresponding calls in vision-api. We start by pulling in the configuration we defined in the code snippet above. Each route constructs a URL using this configuration, we use the Request module to call into the API. We return a response which consists of the response.statusCode and the body of the response:
var logger = require('vision-core').logger
, S = require('string')
, config = require('vision-core').configuration
, request = require('request')
, api = config.get('api:url'),
exports.all = function(req, res){
logger.info('Request.' + req.url);
var userId = req.query.user || req.user.id;
var url = api + '/project?user=' + userId ;
request.get(url, function (error, response, body) {
return res.json(response.statusCode, JSON.parse(body));
});
};
exports.get = function(req, res){
logger.info('Request.' + req.url);
var url = api + '/project/' + req.params.id;
request.get(url, function (error, response, body) {
return res.json(response.statusCode, JSON.parse(body));
});
};
exports.put = function(req, res){
logger.info('Put.' + req.params.id);
if (S(req.body.name).isEmpty() )
return res.json(400, 'Bad Request'),
var url = api + '/project/' + req.params.id;
request.put(url, { form: req.body },
function (error, response, body) {
return res.json(response.statusCode, body);
});
};
exports.post = function(req, res){
logger.info('Post.' + req.body.name);
if (S(req.body.name).isEmpty() )
return res.json(400, 'Bad Request'),
var url = api + '/project/';
request.post(url, { form: req.body },
function (error, response, body) {
var parsed = JSON.parse(body);
res.location('/project/' + parsed._id);
return res.json(response.statusCode, parsed);
});
};
exports.del = function(req, res){
logger.info('Delete.' + req.params.id);
var url = api + '/project/' + req.params.id;
request.del(url, function (error, response, body) {
return res.json(response.statusCode, body);
});
};Let's repeat the same process for our GitHub route ./lib/routes/github.js; removing calls to the GitHub service with calls to the corresponding endpoints on our vision-api project:
var logger = require('vision-core').logger
, config = require('vision-core').configuration
, request = require('request')
, api = config.get('api:url'),
exports.repos = function(req, res){
logger.info('Request.' + req.url);
var url = api + '/project/' + req.params.id + "/repos";
request.get(url, function (error, response, body) {
return res.json(response.statusCode, JSON.parse(body));
});
};
exports.commits = function(req, res){
logger.info('Request.' + req.url);
var url = api + '/project/' + req.params.id + "/commits";
request.get(url, function (error, response, body) {
return res.json(response.statusCode, JSON.parse(body));
});
};
exports.issues = function(req, res){
logger.info('Request.' + req.url);
var url = api + '/project/' + req.params.id + "/issues";
request.get(url, function (error, response, body) {
return res.json(response.statusCode, JSON.parse(body));
});
};Lets update our tests ./test/project.js, ./test/github.js. We now remove anything Mongoose related with direct calls using Request module to vision-api in order to seed test data to MongoDB:
beforeEach(function(done){
var proj = {
name: "test name"
, user: login.user
, token: login.token
, image: "/img/"
, repositories : [ "node-plates" ]
};
var url = api + '/project';
req.post(url, { form: proj },
function (error, response, body) {
id = JSON.parse(body)._id;
done()
});
});
afterEach(function(done){
var url = api + '/project/' + id;
req.del(url, function (error, response, body) {
done()
});
});