commit 3793c0f36305e90d816156735c55b46a6adb007c Author: Jocelyn Badgley (Twipped) Date: Thu Feb 13 09:46:46 2020 -0800 Adding terraform config This currently just redirects to the old bible, but at least everything is up and running diff --git a/terraform/cert.tf b/terraform/cert.tf new file mode 100644 index 0000000..a7bc915 --- /dev/null +++ b/terraform/cert.tf @@ -0,0 +1,38 @@ + +# ----------------------------------------------------------------------------------------------------------- +# Website SSL Certificate + +resource "aws_acm_certificate" "cert" { + domain_name = var.domain + validation_method = "DNS" + + subject_alternative_names = formatlist("%s.%s", + var.subdomains, + var.domain, + ) + + tags = { + Name = "Site Certificate" + Site = var.site + Category = "SSL" + } + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_route53_record" "cert_validation" { + count = length(aws_acm_certificate.cert.subject_alternative_names) + 1 + zone_id = aws_route53_zone.zone.id + ttl = 60 + + name = aws_acm_certificate.cert.domain_validation_options[count.index].resource_record_name + type = aws_acm_certificate.cert.domain_validation_options[count.index].resource_record_type + records = [aws_acm_certificate.cert.domain_validation_options[count.index].resource_record_value] +} + +resource "aws_acm_certificate_validation" "cert" { + certificate_arn = aws_acm_certificate.cert.arn + validation_record_fqdns = aws_route53_record.cert_validation[*].fqdn +} diff --git a/terraform/dns.tf b/terraform/dns.tf new file mode 100644 index 0000000..e171f3a --- /dev/null +++ b/terraform/dns.tf @@ -0,0 +1,65 @@ + + +# ----------------------------------------------------------------------------------------------------------- +# Site DNS Zone and extra domains + +resource "aws_route53_zone" "zone" { + name = var.domain + + tags = { + Site = var.site + Category = "DNS" + } +} + +# ----------------------------------------------------------------------------------------------------------- +# Email Domains + +resource "aws_route53_record" "mail_exchange" { + zone_id = aws_route53_zone.zone.zone_id + name = "" + type = "MX" + ttl = 86400 + + records = [ + "1 ASPMX.L.GOOGLE.COM", + "5 ALT1.ASPMX.L.GOOGLE.COM", + "5 ALT2.ASPMX.L.GOOGLE.COM", + "10 ASPMX2.GOOGLEMAIL.COM", + "10 ASPMX3.GOOGLEMAIL.COM", + ] + +} + +resource "aws_route53_record" "google_mail_verify" { + zone_id = aws_route53_zone.zone.zone_id + name = "" + type = "TXT" + ttl = 300 + + records = [ + "google-site-verification=qidEaa68dfZvcMDRv-pQdLlTSUpF1TJWiwtaVoGK8s8", + ] +} + + +resource "aws_route53_record" "google_mail_secure" { + zone_id = aws_route53_zone.zone.zone_id + name = "" + type = "SPF" + ttl = 86400 + + records = [ + "v=spf1 include:_spf.google.com ~all" + ] +} + +resource "aws_route53_record" "dkim" { + zone_id = aws_route53_zone.zone.zone_id + name = "google._domainkey" + type = "TXT" + ttl = 300 + records = [ + "v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwwirvGkh1h1vMmlK1IEHgs6tlfvkGPv7OLT2yz8hppjTe+sIov8DcBsj3NwotlywotXXgibO5fRJfLHgz0t0eTGeY15c/3K75VnVtKTm4QQ80COU/dCQ1ZbdSmfthEA7w2r0rAEXf20/2J+s8JzCwUidPQfCoYDH+QfSSw\"\"LjOwzSrjBPn+gg2Weh75DxmPHvw1mxA1WD0s+QjZlrLs4hgv41LMJr68Jh5zy+FVRNJAFX1HHVumZDS0StbaDU6r7CvARODQjv+0YMHQRvhDN9LXPp+RGRIegF6ApUM4nEDhDAiM8a/ubUXacX3jMMrSuHgxwhKAk6l0m/LctiNZ943QIDAQAB" + ] +} diff --git a/terraform/files/decorate/index.js b/terraform/files/decorate/index.js new file mode 100644 index 0000000..d4b23ee --- /dev/null +++ b/terraform/files/decorate/index.js @@ -0,0 +1,92 @@ +/* eslint no-console:0 */ +/* global URLSearchParams */ + +const AWS = require('aws-sdk'); +const zlib = require('zlib'); +const util = require('util'); +const path = require('path'); +const {URL} = require('url'); +const s3 = new AWS.S3(); +const { parse: parseLog } = require('cloudfront-log-parser'); +const parseUA = require('ua-parser-js'); +const format = require('date-fns/format'); +const gunzip = util.promisify(zlib.gunzip); +const gzip = util.promisify(zlib.gzip); + +function url (input) { + const { hash, host, hostname, href, origin, password, pathname, port, protocol, search, searchParams, username } = new URL(input); + return { hash, host, hostname, href, origin, password, pathname, port, protocol, search, searchParams, username }; +} + +exports.handler = async (event) => { + // Read options from the event. + console.log('Reading options from event:\n', JSON.stringify(event, null, 2)); + + const Bucket = event.Records[0].s3.bucket.name; + const inputKey = event.Records[0].s3.object.key; + + const file = path.parse(inputKey); + const outputKey = path.format({ ...file, dir: 'Converted', ext: '.json.gz' }); + + const response = await s3.getObject({ Bucket, Key: inputKey }).promise(); + const input = (await gunzip(response.Body)).toString('utf8'); + + const entries = parseLog(input, { format: 'web' }); + + console.log(`Found ${entries.length} rows`); + + const results = entries.map((row) => { + // filter out OPTIONS calls + if (row['cs-method'] === 'OPTIONS') return null; + + // I only care about the pixel hits, nothing else. + if (row['cs-uri-stem'] !== '/i') return null; + + // this isn't an analytics event + if (row['cs-referer'] === '-') return null; + + row = Object.fromEntries(Object.entries(row).map(([ k, v ]) => [ k.replace(/-/g, '_'), v ])); + + const query = (row.cs_uri_query === '-') + ? {} + : Object.fromEntries(new URLSearchParams(row.cs_uri_query)) + ; + + // we didn't get analytics data from this load, ignore it + if (!query.start) return null; + + const useragent = parseUA(row.cs_user_agent); + const { referer } = query; + + const sessionStart = Number(query.start); + const sessionEnd = query.end === 'null' ? 0 : Number(query.end); + const duration = sessionEnd > sessionStart ? Math.floor((sessionEnd - sessionStart) / 1000) : null; + + return JSON.stringify({ + dts: `${row.date} ${row.time}`, + url: url(row.cs_referer), + referer: url(query.referer), + client_start: format(new Date(sessionStart), 'yyyy-MM-dd HH:mm:ss'), + client_end: sessionEnd ? format(new Date(sessionStart), 'yyyy-MM-dd HH:mm:ss') : null, + duration, + useragent, + query, + original: row, + }); + }).filter(Boolean); + + if (!results.length) { + console.log('No results to save'); + return; + } + + console.log('Writing new file to ' + outputKey); + + await s3.putObject({ + Bucket, + Key: outputKey, + Body: await gzip(Buffer.from(results.join('\n'))), + ContentType: 'application/gzip', + }).promise(); + +}; diff --git a/terraform/files/decorate/package-lock.json b/terraform/files/decorate/package-lock.json new file mode 100644 index 0000000..748c795 --- /dev/null +++ b/terraform/files/decorate/package-lock.json @@ -0,0 +1,23 @@ +{ + "name": "decorate", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "cloudfront-log-parser": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/cloudfront-log-parser/-/cloudfront-log-parser-1.1.0.tgz", + "integrity": "sha512-g1lxh8aW5ZrQ7/loX2/vLzz4SWefQhSvZw++wgoIx3aEugXHKyfYaWOXGS4pNNp9hi7JcXITxLYqBI2FY+jtgA==" + }, + "date-fns": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.9.0.tgz", + "integrity": "sha512-khbFLu/MlzLjEzy9Gh8oY1hNt/Dvxw3J6Rbc28cVoYWQaC1S3YI4xwkF9ZWcjDLscbZlY9hISMr66RFzZagLsA==" + }, + "ua-parser-js": { + "version": "0.7.21", + "resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz", + "integrity": "sha512-+O8/qh/Qj8CgC6eYBVBykMrNtp5Gebn4dlGD/kKXVkJNDwyrAwSIqwz8CDf+tsAIWVycKcku6gIXJ0qwx/ZXaQ==" + } + } +} diff --git a/terraform/files/decorate/package.json b/terraform/files/decorate/package.json new file mode 100644 index 0000000..7f6a5ec --- /dev/null +++ b/terraform/files/decorate/package.json @@ -0,0 +1,19 @@ +{ + "name": "decorate", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Jocelyn Badgley (http://twipped.com/)", + "license": "MIT", + "dependencies": { + "cloudfront-log-parser": "~1.1.0", + "date-fns": "~2.9.0", + "ua-parser-js": "~0.7.21" + }, + "engines": { + "node": ">=12.14.0" + } +} diff --git a/terraform/files/i.gif b/terraform/files/i.gif new file mode 100644 index 0000000..3914955 Binary files /dev/null and b/terraform/files/i.gif differ diff --git a/terraform/files/index_redirect.js b/terraform/files/index_redirect.js new file mode 100644 index 0000000..c0805ce --- /dev/null +++ b/terraform/files/index_redirect.js @@ -0,0 +1,22 @@ +// This is a solution for the problem with s3 websites and subfolders www.site.com/folder/ will not auto +// redirect to www.site.com/folder/index.html like most other modern web servers +// This should be deployed as a Lambda@Edge connected to the CloudFront Distribution +// Only Node.js 10.x Runtime supports Lambda@Edge for right now, we have to wait for AWS to support 12x and beyond + +exports.handler = async (event, context) => { + /* + * Expand S3 request to have index.html if it ends in / + */ + const request = event.Records[0].cf.request; + if ((request.uri !== '/') /* Not the root object, which redirects properly */ + && (request.uri.endsWith('/') /* Folder with slash */ + || (request.uri.lastIndexOf('.') < request.uri.lastIndexOf('/')) /* Most likely a folder, it has no extension (heuristic) */ + )) { + if (request.uri.endsWith('/')) { + request.uri = request.uri.concat('index.html'); + } else { + request.uri = request.uri.concat('/index.html'); + } + } + return request; +}; diff --git a/terraform/files/table.sql b/terraform/files/table.sql new file mode 100644 index 0000000..94e9e54 --- /dev/null +++ b/terraform/files/table.sql @@ -0,0 +1,23 @@ +CREATE EXTERNAL TABLE gdbible.events ( + dts TIMESTAMP, + page STRING, + client_start TIMESTAMP, + client_end TIMESTAMP, + duration INT, + useragent struct< + browser: struct< name:STRING, version:STRING, major: INT >, + os: struct< name:STRING, version:STRING > + >, + `query` struct< + tid: STRING, + page_height:INT, + viewport_height:INT, + max_scroll:INT, + viewed:INT, + language:STRING, + referrer:STRING + >, + original struct +) +ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe' +LOCATION 's3://gdbible-analytics/Converted' diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..01a38f3 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,99 @@ + + +# ----------------------------------------------------------------------------------------------------------- +# IAM Role for Redirect Lambda + +resource "aws_iam_role" "lambda_redirect" { + name = "${var.site}-lambda-redirect-role" + assume_role_policy = < + + +EOF +} + +# ----------------------------------------------------------------------------------------------------------- +# Cloudfront Configuration + +resource "aws_cloudfront_distribution" "site" { + origin { + domain_name = aws_s3_bucket.temp_redirect.bucket_regional_domain_name + origin_id = "S3-Website-${aws_s3_bucket.src.website_endpoint}" + + custom_origin_config { + origin_protocol_policy = "http-only" + http_port = "80" + https_port = "443" + origin_ssl_protocols = ["SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"] + } + } + + enabled = true + is_ipv6_enabled = true + default_root_object = "index.html" + + aliases = [ + var.domain, + "www.${var.domain}" + ] + + default_cache_behavior { + allowed_methods = ["GET", "HEAD"] + cached_methods = ["GET", "HEAD"] + target_origin_id = "S3-Website-${aws_s3_bucket.src.website_endpoint}" + + forwarded_values { + query_string = false + + cookies { + forward = "none" + } + } + + # lambda_function_association { + # event_type = "origin-request" + # lambda_arn = aws_lambda_function.index_redirect.qualified_arn + # include_body = false + # } + + viewer_protocol_policy = "redirect-to-https" + min_ttl = 0 + default_ttl = 86400 + max_ttl = 31536000 + } + + restrictions { + geo_restriction { + restriction_type = "none" + } + } + + viewer_certificate { + acm_certificate_arn = aws_acm_certificate.cert.arn + ssl_support_method = "sni-only" + minimum_protocol_version = "TLSv1.1_2016" + } + + # viewer_certificate { + # cloudfront_default_certificate = true + # } + + tags = { + Name = "Main Site" + Site = var.site + } +} + +# ----------------------------------------------------------------------------------------------------------- +# Domains + +resource "aws_route53_record" "site" { + name = var.domain + zone_id = aws_route53_zone.zone.zone_id + type = "A" + + alias { + name = aws_cloudfront_distribution.site.domain_name + zone_id = aws_cloudfront_distribution.site.hosted_zone_id + evaluate_target_health = false + } +} + +resource "aws_route53_record" "www" { + name = "www.${var.domain}" + zone_id = aws_route53_zone.zone.zone_id + type = "A" + + alias { + name = aws_cloudfront_distribution.site.domain_name + zone_id = aws_cloudfront_distribution.site.hosted_zone_id + evaluate_target_health = false + } +} + +# ----------------------------------------------------------------------------------------------------------- +# Lambda Subdirectory index.html Redirect + +# data "archive_file" "index_redirect" { +# type = "zip" +# output_path = "${path.module}/files/index_redirect.js.zip" +# source_file = "${path.module}/files/index_redirect.js" +# } + +# resource "aws_lambda_function" "index_redirect" { +# description = "index.html subdirectory redirect" +# filename = "${path.module}/files/index_redirect.js.zip" +# function_name = "folder-index-redirect" +# handler = "index_redirect.handler" +# source_code_hash = data.archive_file.index_redirect.output_base64sha256 +# publish = true +# role = aws_iam_role.lambda_redirect.arn +# runtime = "nodejs12.x" + +# tags = { +# Name = "${var.site}-index-redirect" +# Site = var.site +# } +# }