| 1 |
/* |
| 2 |
Copyright 2015 Google Inc. All rights reserved. |
| 3 |
|
| 4 |
Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 |
you may not use this file except in compliance with the License. |
| 6 |
You may obtain a copy of the License at |
| 7 |
|
| 8 |
http://www.apache.org/licenses/LICENSE-2.0 |
| 9 |
|
| 10 |
Unless required by applicable law or agreed to in writing, software |
| 11 |
distributed under the License is distributed on an "AS IS" BASIS, |
| 12 |
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 |
See the License for the specific language governing permissions and |
| 14 |
limitations under the License. |
| 15 |
*/ |
| 16 |
|
| 17 |
// Import some libraries |
| 18 |
local packer = import "packer.libsonnet"; |
| 19 |
local terraform = import "terraform.libsonnet"; |
| 20 |
local cassandra = import "cassandra.libsonnet"; |
| 21 |
|
| 22 |
// Credentials file (don't commit this!) |
| 23 |
local credentials = import "credentials.libsonnet"; |
| 24 |
|
| 25 |
|
| 26 |
function(ssh_username) { |
| 27 |
|
| 28 |
/////////////////////////// |
| 29 |
// GENERAL CONFIGURATION // |
| 30 |
/////////////////////////// |
| 31 |
|
| 32 |
local tilegenPort = 8080, |
| 33 |
|
| 34 |
local cassandraUser = "fractal", // Password in credentials import. |
| 35 |
local cassandraKeyspace = "fractal", |
| 36 |
local cassandraReplication = "{ 'class' : 'SimpleStrategy', 'replication_factor' : 2 }", |
| 37 |
|
| 38 |
// These are the nodes the application server attempts to use (client-side load balancing). |
| 39 |
local cassandraNodes = ["db1", "db2", "db3", "db4", "db5"], |
| 40 |
|
| 41 |
// The Cassandra configuration file we use for this service. |
| 42 |
local cassandraConf = cassandra.conf { |
| 43 |
cluster_name: "Fractal Cluster", |
| 44 |
rpc_address:: null, // Unset by making it hidden (::). |
| 45 |
listen_address:: null, // Unset by making it hidden (::). |
| 46 |
authenticator: "PasswordAuthenticator", |
| 47 |
seed_provider: [ |
| 48 |
{ |
| 49 |
class_name: "org.apache.cassandra.locator.SimpleSeedProvider", |
| 50 |
parameters: [{ seeds: std.join(", ", cassandraNodes) }], |
| 51 |
}, |
| 52 |
], |
| 53 |
}, |
| 54 |
|
| 55 |
// Create an initial (empty) database for storing 'discovered' fractal coordinates. |
| 56 |
local cql_insert(uuid, x, y, l, n) = |
| 57 |
"INSERT INTO discoveries (Date, TimeId, X, Y, L, Text) " |
| 58 |
+ ("VALUES ('FIXED', %s, %s, %s, %s, '%s');" % [uuid, x, y, l, n]), |
| 59 |
|
| 60 |
local cassandraInitCql = [ |
| 61 |
"CREATE USER %s WITH PASSWORD '%s';" % [cassandraUser, credentials.cassandraUserPass], |
| 62 |
"CREATE KEYSPACE %s WITH REPLICATION = %s;" % [cassandraKeyspace, cassandraReplication], |
| 63 |
"USE %s;" % cassandraKeyspace, |
| 64 |
"CREATE TABLE discoveries(" |
| 65 |
+ "Date TEXT, TimeId TIMEUUID, Text TEXT, X FLOAT, Y FLOAT, L INT, " |
| 66 |
+ "PRIMARY KEY(Date, TimeId));", |
| 67 |
cql_insert("18063880-5a4d-11e4-ada4-247703d0f194", "0", "0", "0", "Zoomed Out"), |
| 68 |
cql_insert("66b6d100-5a53-11e4-aa05-247703d0f194", |
| 69 |
"-1.21142578125", "0.3212890625", "4", "Lightning"), |
| 70 |
cql_insert("77ffdd80-5a53-11e4-8ccf-247703d0f194", |
| 71 |
"-1.7568359375", "-0.0009765625", "5", "Self-similarity"), |
| 72 |
cql_insert("7fbf8200-5a53-11e4-804a-247703d0f194", |
| 73 |
"0.342529296875", "0.419189453125", "5", "Windmills"), |
| 74 |
cql_insert("9ae7bd00-5a66-11e4-9c66-247703d0f194", |
| 75 |
"-1.48309979046093", "0.00310595797955671", "39", "Star"), |
| 76 |
cql_insert("75fe4480-5a7c-11e4-a747-247703d0f194", |
| 77 |
"-0.244976043701172", "0.716987609863281", "10", "Baroque"), |
| 78 |
cql_insert("abf70380-5b24-11e4-8a46-247703d0f194", |
| 79 |
"-1.74749755859375", "0.009002685546875", "9", "Hairy windmills"), |
| 80 |
], |
| 81 |
|
| 82 |
// Configuration shared by the application server and tile generation nodes. |
| 83 |
local ApplicationConf = { |
| 84 |
width: 256, |
| 85 |
height: 256, |
| 86 |
thumb_width: 64, |
| 87 |
thumb_height: 64, |
| 88 |
iters: 200, |
| 89 |
database: cassandraKeyspace, |
| 90 |
tilegen: "${google_compute_address.tilegen.address}:%d" % tilegenPort, |
| 91 |
db_endpoints: cassandraNodes, |
| 92 |
}, |
| 93 |
|
| 94 |
/////////////////////////// |
| 95 |
// PACKER CONFIGURATIONS // |
| 96 |
/////////////////////////// |
| 97 |
|
| 98 |
// Some config used in every Packer image we create. |
| 99 |
local ImageMixin = { |
| 100 |
project_id: credentials.project, |
| 101 |
account_file: "service_account_key.json", |
| 102 |
|
| 103 |
// For debugging: |
| 104 |
local network_debug = ["traceroute", "lsof", "iptraf", "tcpdump", "host", "dnsutils"], |
| 105 |
aptPackages+: ["vim", "git", "psmisc", "screen", "strace"] + network_debug, |
| 106 |
sshUsername: ssh_username, |
| 107 |
}, |
| 108 |
|
| 109 |
// Frontend image. |
| 110 |
"appserv.packer.json": packer.GcpDebianNginxUwsgiFlaskImage + ImageMixin { |
| 111 |
name: "appserv-v20150430-2145", |
| 112 |
module: "main", // Entrypoint in the Python code. |
| 113 |
pipPackages+: ["httplib2", "cassandra-driver", "blist"], |
| 114 |
uwsgiConf+: { lazy: "true" }, // cassandra-driver does not survive fork() |
| 115 |
// Copy website content and code. |
| 116 |
provisioners+: [ |
| 117 |
packer.File { |
| 118 |
source: "appserv", |
| 119 |
destination: "/tmp/", |
| 120 |
}, |
| 121 |
packer.RootShell { inline: [ |
| 122 |
"mv /tmp/appserv/* /var/www/", |
| 123 |
"chown -R www-data.www-data /var/www/*", |
| 124 |
] }, |
| 125 |
], |
| 126 |
}, |
| 127 |
|
| 128 |
// The Cassandra image is basic, but more configuration is done at deployment time. |
| 129 |
"cassandra.packer.json": cassandra.GcpDebianImage + ImageMixin { |
| 130 |
name: "cassandra-v20150430-2145", |
| 131 |
rootPassword: credentials.cassandraRootPass, |
| 132 |
clusterName: cassandraConf.cluster_name, |
| 133 |
}, |
| 134 |
|
| 135 |
// Tile Generation node runs a C++ program to generate PNG tiles for the fractal. |
| 136 |
"tilegen.packer.json": packer.GcpDebianNginxUwsgiFlaskImage + ImageMixin { |
| 137 |
name: "tilegen-v20150430-2145", |
| 138 |
module: "mandelbrot_service", |
| 139 |
|
| 140 |
aptPackages+: ["g++", "libpng-dev"], |
| 141 |
|
| 142 |
port: tilegenPort, |
| 143 |
|
| 144 |
// Copy the flask handlers and also build the C++ executable. |
| 145 |
provisioners+: [ |
| 146 |
packer.File { |
| 147 |
source: "tilegen", |
| 148 |
destination: "/tmp/", |
| 149 |
}, |
| 150 |
packer.RootShell { inline: [ |
| 151 |
"mv /tmp/tilegen/* /var/www/", |
| 152 |
"chown -R www-data.www-data /var/www/*", |
| 153 |
] }, |
| 154 |
packer.RootShell { inline: [ |
| 155 |
"g++ -Wall -Wextra -ansi -pedantic -O3 -ffast-math -g " |
| 156 |
+ "/var/www/mandelbrot.cpp -lpng -o /var/www/mandelbrot", |
| 157 |
] }, |
| 158 |
], |
| 159 |
}, |
| 160 |
|
| 161 |
|
| 162 |
///////////////////////////// |
| 163 |
// TERRAFORM CONFIGURATION // |
| 164 |
///////////////////////////// |
| 165 |
|
| 166 |
"terraform.tf": { |
| 167 |
|
| 168 |
// How to contact the Google Cloud Platform APIs. |
| 169 |
provider: { |
| 170 |
google: { |
| 171 |
account_file: "service_account_key.json", |
| 172 |
project: credentials.project, |
| 173 |
region: "us-central1", |
| 174 |
}, |
| 175 |
}, |
| 176 |
|
| 177 |
// The deployed resources. |
| 178 |
resource: { |
| 179 |
|
| 180 |
// Instances are assigned zones on a round robin scheme. |
| 181 |
local zone(hash) = |
| 182 |
local arr = [ |
| 183 |
"us-central1-c", |
| 184 |
"us-central1-b", |
| 185 |
"us-central1-f", |
| 186 |
]; |
| 187 |
arr[hash % std.length(arr)], |
| 188 |
|
| 189 |
// The internal subnet. |
| 190 |
google_compute_network: { |
| 191 |
fractal: { |
| 192 |
name: "fractal", |
| 193 |
ipv4_range: "10.0.0.0/16", |
| 194 |
}, |
| 195 |
}, |
| 196 |
|
| 197 |
// Publicly visible static ip addresses. |
| 198 |
google_compute_address: { |
| 199 |
appserv: { name: "appserv" }, |
| 200 |
tilegen: { name: "tilegen" }, |
| 201 |
}, |
| 202 |
|
| 203 |
// The next 3 resource types configure load balancing for the appserv and tilegen. |
| 204 |
google_compute_http_health_check: { |
| 205 |
appserv: { |
| 206 |
name: "appserv", |
| 207 |
port: 80, |
| 208 |
}, |
| 209 |
tilegen: { |
| 210 |
name: "tilegen", |
| 211 |
port: tilegenPort, |
| 212 |
}, |
| 213 |
}, |
| 214 |
|
| 215 |
google_compute_target_pool: { |
| 216 |
appserv: { |
| 217 |
name: "appserv", |
| 218 |
health_checks: ["${google_compute_http_health_check.appserv.name}"], |
| 219 |
instances: ["%s/appserv%d" % [zone(k), k] for k in [1, 2, 3]], |
| 220 |
}, |
| 221 |
tilegen: { |
| 222 |
name: "tilegen", |
| 223 |
health_checks: ["${google_compute_http_health_check.tilegen.name}"], |
| 224 |
instances: ["%s/tilegen%d" % [zone(k), k] for k in [1, 2, 3, 4, 5]], |
| 225 |
}, |
| 226 |
}, |
| 227 |
|
| 228 |
google_compute_forwarding_rule: { |
| 229 |
appserv: { |
| 230 |
ip_address: "${google_compute_address.appserv.address}", |
| 231 |
name: "appserv", |
| 232 |
target: "${google_compute_target_pool.appserv.self_link}", |
| 233 |
port_range: "80", |
| 234 |
}, |
| 235 |
tilegen: { |
| 236 |
ip_address: "${google_compute_address.tilegen.address}", |
| 237 |
name: "tilegen", |
| 238 |
target: "${google_compute_target_pool.tilegen.self_link}", |
| 239 |
port_range: tilegenPort, |
| 240 |
}, |
| 241 |
}, |
| 242 |
|
| 243 |
// Open ports for the various services, to instances (identified by tags) |
| 244 |
google_compute_firewall: { |
| 245 |
local NetworkMixin = { network: "${google_compute_network.fractal.name}" }, |
| 246 |
ssh: terraform.GcpFirewallSsh + NetworkMixin { name: "ssh" }, |
| 247 |
appserv: terraform.GcpFirewallHttp + NetworkMixin { name: "appserv" }, |
| 248 |
tilegen: terraform.GcpFirewallHttp + NetworkMixin { name: "tilegen", port: tilegenPort }, |
| 249 |
cassandra: cassandra.GcpFirewall + NetworkMixin { name: "cassandra" }, |
| 250 |
gossip: cassandra.GcpFirewallGossip + NetworkMixin { name: "gossip" }, |
| 251 |
}, |
| 252 |
|
| 253 |
// All our instances share this configuration. |
| 254 |
local FractalInstance(zone_hash) = terraform.GcpInstance { |
| 255 |
network_interface: [super.network_interface[0] { network: "${google_compute_network.fractal.name}" }], |
| 256 |
tags+: ["fractal"], |
| 257 |
zone: zone(zone_hash), |
| 258 |
scopes+: ["devstorage.full_control"], |
| 259 |
}, |
| 260 |
|
| 261 |
// The various kinds of Cassandra instances all share this basic configuration. |
| 262 |
local CassandraInstance(zone_hash) = FractalInstance(zone_hash) { |
| 263 |
image: "cassandra-v20150430-2145", |
| 264 |
machine_type: "n1-standard-1", |
| 265 |
tags+: ["fractal-db", "cassandra-server"], |
| 266 |
user:: cassandraUser, |
| 267 |
userPass:: credentials.cassandraUserPass, |
| 268 |
rootPass:: credentials.cassandraRootPass, |
| 269 |
conf:: cassandraConf, |
| 270 |
}, |
| 271 |
|
| 272 |
google_compute_instance: { |
| 273 |
|
| 274 |
// The application server instances have database credentials and the address of the |
| 275 |
// tilegen loadbalancer. This is all stored in a conf.json, read by the python |
| 276 |
// code. |
| 277 |
["appserv" + k]: FractalInstance(k) { |
| 278 |
name: "appserv" + k, |
| 279 |
image: "appserv-v20150430-2145", |
| 280 |
conf:: ApplicationConf { |
| 281 |
database_name: cassandraKeyspace, |
| 282 |
database_user: cassandraUser, |
| 283 |
database_pass: credentials.cassandraUserPass, |
| 284 |
}, |
| 285 |
tags+: ["fractal-appserv", "http-server"], |
| 286 |
startup_script+: [self.addFile(self.conf, "/var/www/conf.json")], |
| 287 |
} |
| 288 |
for k in [1, 2, 3] |
| 289 |
|
| 290 |
} + { |
| 291 |
|
| 292 |
// Bootstrapping the Cassandra database is a little subtle. We bring up 3 nodes |
| 293 |
// in parallel, one of which is special and creates the initial database. The other |
| 294 |
// two join it. Note the two different mixins used to control that behavior. |
| 295 |
|
| 296 |
db1: CassandraInstance(1) + cassandra.GcpStarterMixin { |
| 297 |
name: "db1", |
| 298 |
// Replication of the system_auth table (user credentials). |
| 299 |
authReplication:: cassandraReplication, |
| 300 |
// The CQL code is used to bootstrap the database. |
| 301 |
initCql:: cassandraInitCql, |
| 302 |
}, |
| 303 |
|
| 304 |
// TopUpMixin creates an empty Cassandra node which can join an existing cluster. |
| 305 |
db2: CassandraInstance(2) + cassandra.GcpTopUpMixin { |
| 306 |
name: "db2", |
| 307 |
}, |
| 308 |
|
| 309 |
db3: CassandraInstance(3) + cassandra.GcpTopUpMixin { |
| 310 |
name: "db3", |
| 311 |
}, |
| 312 |
|
| 313 |
// To increase the size of the cluster, these can be used. Bring them up one by one |
| 314 |
// verifying the state of the database each time by logging onto an existing db node |
| 315 |
// and running "nodetool status fractal". To reduce the size of the cluster, use |
| 316 |
// nodetool -h $HOST decommission and then remove the node from this configuration. |
| 317 |
// If a node is removed without decommissioning, fix the cluster with nodetool |
| 318 |
// removenode <UUID>. It is possible to completely recycle the nodes (including the |
| 319 |
// first node) without data loss as long as nodetool is used judiciously. |
| 320 |
/* |
| 321 |
db4: CassandraInstance(4) + cassandra.GcpTopUpMixin { |
| 322 |
name: "db4", |
| 323 |
}, |
| 324 |
*/ |
| 325 |
|
| 326 |
} + { |
| 327 |
// The tile generation instances are similar to the application server ones, but do |
| 328 |
// not require database credentials so these are omitted for security. |
| 329 |
["tilegen" + k]: FractalInstance(k) { |
| 330 |
name: "tilegen" + k, |
| 331 |
image: "tilegen-v20150430-2145", |
| 332 |
tags+: ["fractal-tilegen", "http-server"], |
| 333 |
startup_script+: [self.addFile(ApplicationConf, "/var/www/conf.json")], |
| 334 |
} |
| 335 |
for k in [1, 2, 3, 4] |
| 336 |
}, |
| 337 |
|
| 338 |
google_dns_managed_zone: { |
| 339 |
"fractaldemo-com": { |
| 340 |
name: "fractaldemo-com", |
| 341 |
dns_name: credentials.dnsPrefix + "fractaldemo.com.", |
| 342 |
description: "Fractal Demo DNS Zone", |
| 343 |
}, |
| 344 |
}, |
| 345 |
|
| 346 |
local instances = self.google_compute_instance, |
| 347 |
local addresses = self.google_compute_address, |
| 348 |
|
| 349 |
local DnsRecord = { |
| 350 |
managed_zone: "${google_dns_managed_zone.fractaldemo-com.name}", |
| 351 |
type: "A", |
| 352 |
ttl: 300, |
| 353 |
}, |
| 354 |
|
| 355 |
google_dns_record_set: { |
| 356 |
[name]: DnsRecord { |
| 357 |
name: name + ".${google_dns_managed_zone.fractaldemo-com.dns_name}", |
| 358 |
rrdatas: ["${google_compute_address." + name + ".address}"], |
| 359 |
} for name in std.objectFields(addresses) |
| 360 |
} + { |
| 361 |
[name]: DnsRecord { |
| 362 |
name: name + ".${google_dns_managed_zone.fractaldemo-com.dns_name}", |
| 363 |
rrdatas: ["${google_compute_instance." + name + ".network_interface.0.access_config.0.nat_ip}"], |
| 364 |
} for name in std.objectFields(instances) |
| 365 |
} + { |
| 366 |
www: { |
| 367 |
managed_zone: "${google_dns_managed_zone.fractaldemo-com.name}", |
| 368 |
name: "www.${google_dns_managed_zone.fractaldemo-com.dns_name}", |
| 369 |
type: "CNAME", |
| 370 |
ttl: 300, |
| 371 |
rrdatas: ["appserv.${google_dns_managed_zone.fractaldemo-com.dns_name}"], |
| 372 |
}, |
| 373 |
zone: { |
| 374 |
managed_zone: "${google_dns_managed_zone.fractaldemo-com.name}", |
| 375 |
name: "${google_dns_managed_zone.fractaldemo-com.dns_name}", |
| 376 |
type: "A", |
| 377 |
ttl: 300, |
| 378 |
rrdatas: ["${google_compute_address.appserv.address}"], |
| 379 |
}, |
| 380 |
}, |
| 381 |
|
| 382 |
}, |
| 383 |
|
| 384 |
output: { |
| 385 |
frontend: { value: "${google_compute_address.appserv.address}" }, |
| 386 |
}, |
| 387 |
|
| 388 |
}, // terraform.tf |
| 389 |
|
| 390 |
} |