Adding terraform config

This currently just redirects to the old bible, but at least everything is up and running
This commit is contained in:
Jocelyn Badgley (Twipped) 2020-02-13 09:46:46 -08:00
commit 3793c0f363
14 changed files with 877 additions and 0 deletions

38
terraform/cert.tf Normal file
View File

@ -0,0 +1,38 @@
# -----------------------------------------------------------------------------------------------------------
# Website SSL Certificate
resource "aws_acm_certificate" "cert" {
domain_name = var.domain
validation_method = "DNS"
subject_alternative_names = formatlist("%s.%s",
var.subdomains,
var.domain,
)
tags = {
Name = "Site Certificate"
Site = var.site
Category = "SSL"
}
lifecycle {
create_before_destroy = true
}
}
resource "aws_route53_record" "cert_validation" {
count = length(aws_acm_certificate.cert.subject_alternative_names) + 1
zone_id = aws_route53_zone.zone.id
ttl = 60
name = aws_acm_certificate.cert.domain_validation_options[count.index].resource_record_name
type = aws_acm_certificate.cert.domain_validation_options[count.index].resource_record_type
records = [aws_acm_certificate.cert.domain_validation_options[count.index].resource_record_value]
}
resource "aws_acm_certificate_validation" "cert" {
certificate_arn = aws_acm_certificate.cert.arn
validation_record_fqdns = aws_route53_record.cert_validation[*].fqdn
}

65
terraform/dns.tf Normal file
View File

@ -0,0 +1,65 @@
# -----------------------------------------------------------------------------------------------------------
# Site DNS Zone and extra domains
resource "aws_route53_zone" "zone" {
name = var.domain
tags = {
Site = var.site
Category = "DNS"
}
}
# -----------------------------------------------------------------------------------------------------------
# Email Domains
resource "aws_route53_record" "mail_exchange" {
zone_id = aws_route53_zone.zone.zone_id
name = ""
type = "MX"
ttl = 86400
records = [
"1 ASPMX.L.GOOGLE.COM",
"5 ALT1.ASPMX.L.GOOGLE.COM",
"5 ALT2.ASPMX.L.GOOGLE.COM",
"10 ASPMX2.GOOGLEMAIL.COM",
"10 ASPMX3.GOOGLEMAIL.COM",
]
}
resource "aws_route53_record" "google_mail_verify" {
zone_id = aws_route53_zone.zone.zone_id
name = ""
type = "TXT"
ttl = 300
records = [
"google-site-verification=qidEaa68dfZvcMDRv-pQdLlTSUpF1TJWiwtaVoGK8s8",
]
}
resource "aws_route53_record" "google_mail_secure" {
zone_id = aws_route53_zone.zone.zone_id
name = ""
type = "SPF"
ttl = 86400
records = [
"v=spf1 include:_spf.google.com ~all"
]
}
resource "aws_route53_record" "dkim" {
zone_id = aws_route53_zone.zone.zone_id
name = "google._domainkey"
type = "TXT"
ttl = 300
records = [
"v=DKIM1; k=rsa; p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwwirvGkh1h1vMmlK1IEHgs6tlfvkGPv7OLT2yz8hppjTe+sIov8DcBsj3NwotlywotXXgibO5fRJfLHgz0t0eTGeY15c/3K75VnVtKTm4QQ80COU/dCQ1ZbdSmfthEA7w2r0rAEXf20/2J+s8JzCwUidPQfCoYDH+QfSSw\"\"LjOwzSrjBPn+gg2Weh75DxmPHvw1mxA1WD0s+QjZlrLs4hgv41LMJr68Jh5zy+FVRNJAFX1HHVumZDS0StbaDU6r7CvARODQjv+0YMHQRvhDN9LXPp+RGRIegF6ApUM4nEDhDAiM8a/ubUXacX3jMMrSuHgxwhKAk6l0m/LctiNZ943QIDAQAB"
]
}

View File

@ -0,0 +1,92 @@
/* eslint no-console:0 */
/* global URLSearchParams */
const AWS = require('aws-sdk');
const zlib = require('zlib');
const util = require('util');
const path = require('path');
const {URL} = require('url');
const s3 = new AWS.S3();
const { parse: parseLog } = require('cloudfront-log-parser');
const parseUA = require('ua-parser-js');
const format = require('date-fns/format');
const gunzip = util.promisify(zlib.gunzip);
const gzip = util.promisify(zlib.gzip);
function url (input) {
const { hash, host, hostname, href, origin, password, pathname, port, protocol, search, searchParams, username } = new URL(input);
return { hash, host, hostname, href, origin, password, pathname, port, protocol, search, searchParams, username };
}
exports.handler = async (event) => {
// Read options from the event.
console.log('Reading options from event:\n', JSON.stringify(event, null, 2));
const Bucket = event.Records[0].s3.bucket.name;
const inputKey = event.Records[0].s3.object.key;
const file = path.parse(inputKey);
const outputKey = path.format({ ...file, dir: 'Converted', ext: '.json.gz' });
const response = await s3.getObject({ Bucket, Key: inputKey }).promise();
const input = (await gunzip(response.Body)).toString('utf8');
const entries = parseLog(input, { format: 'web' });
console.log(`Found ${entries.length} rows`);
const results = entries.map((row) => {
// filter out OPTIONS calls
if (row['cs-method'] === 'OPTIONS') return null;
// I only care about the pixel hits, nothing else.
if (row['cs-uri-stem'] !== '/i') return null;
// this isn't an analytics event
if (row['cs-referer'] === '-') return null;
row = Object.fromEntries(Object.entries(row).map(([ k, v ]) => [ k.replace(/-/g, '_'), v ]));
const query = (row.cs_uri_query === '-')
? {}
: Object.fromEntries(new URLSearchParams(row.cs_uri_query))
;
// we didn't get analytics data from this load, ignore it
if (!query.start) return null;
const useragent = parseUA(row.cs_user_agent);
const { referer } = query;
const sessionStart = Number(query.start);
const sessionEnd = query.end === 'null' ? 0 : Number(query.end);
const duration = sessionEnd > sessionStart ? Math.floor((sessionEnd - sessionStart) / 1000) : null;
return JSON.stringify({
dts: `${row.date} ${row.time}`,
url: url(row.cs_referer),
referer: url(query.referer),
client_start: format(new Date(sessionStart), 'yyyy-MM-dd HH:mm:ss'),
client_end: sessionEnd ? format(new Date(sessionStart), 'yyyy-MM-dd HH:mm:ss') : null,
duration,
useragent,
query,
original: row,
});
}).filter(Boolean);
if (!results.length) {
console.log('No results to save');
return;
}
console.log('Writing new file to ' + outputKey);
await s3.putObject({
Bucket,
Key: outputKey,
Body: await gzip(Buffer.from(results.join('\n'))),
ContentType: 'application/gzip',
}).promise();
};

View File

@ -0,0 +1,23 @@
{
"name": "decorate",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"cloudfront-log-parser": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/cloudfront-log-parser/-/cloudfront-log-parser-1.1.0.tgz",
"integrity": "sha512-g1lxh8aW5ZrQ7/loX2/vLzz4SWefQhSvZw++wgoIx3aEugXHKyfYaWOXGS4pNNp9hi7JcXITxLYqBI2FY+jtgA=="
},
"date-fns": {
"version": "2.9.0",
"resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.9.0.tgz",
"integrity": "sha512-khbFLu/MlzLjEzy9Gh8oY1hNt/Dvxw3J6Rbc28cVoYWQaC1S3YI4xwkF9ZWcjDLscbZlY9hISMr66RFzZagLsA=="
},
"ua-parser-js": {
"version": "0.7.21",
"resolved": "https://registry.npmjs.org/ua-parser-js/-/ua-parser-js-0.7.21.tgz",
"integrity": "sha512-+O8/qh/Qj8CgC6eYBVBykMrNtp5Gebn4dlGD/kKXVkJNDwyrAwSIqwz8CDf+tsAIWVycKcku6gIXJ0qwx/ZXaQ=="
}
}
}

View File

@ -0,0 +1,19 @@
{
"name": "decorate",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "Jocelyn Badgley <joc@twipped.com> (http://twipped.com/)",
"license": "MIT",
"dependencies": {
"cloudfront-log-parser": "~1.1.0",
"date-fns": "~2.9.0",
"ua-parser-js": "~0.7.21"
},
"engines": {
"node": ">=12.14.0"
}
}

BIN
terraform/files/i.gif Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 49 B

View File

@ -0,0 +1,22 @@
// This is a solution for the problem with s3 websites and subfolders www.site.com/folder/ will not auto
// redirect to www.site.com/folder/index.html like most other modern web servers
// This should be deployed as a Lambda@Edge connected to the CloudFront Distribution
// Only Node.js 10.x Runtime supports Lambda@Edge for right now, we have to wait for AWS to support 12x and beyond
exports.handler = async (event, context) => {
/*
* Expand S3 request to have index.html if it ends in /
*/
const request = event.Records[0].cf.request;
if ((request.uri !== '/') /* Not the root object, which redirects properly */
&& (request.uri.endsWith('/') /* Folder with slash */
|| (request.uri.lastIndexOf('.') < request.uri.lastIndexOf('/')) /* Most likely a folder, it has no extension (heuristic) */
)) {
if (request.uri.endsWith('/')) {
request.uri = request.uri.concat('index.html');
} else {
request.uri = request.uri.concat('/index.html');
}
}
return request;
};

23
terraform/files/table.sql Normal file
View File

@ -0,0 +1,23 @@
CREATE EXTERNAL TABLE gdbible.events (
dts TIMESTAMP,
page STRING,
client_start TIMESTAMP,
client_end TIMESTAMP,
duration INT,
useragent struct<
browser: struct< name:STRING, version:STRING, major: INT >,
os: struct< name:STRING, version:STRING >
>,
`query` struct<
tid: STRING,
page_height:INT,
viewport_height:INT,
max_scroll:INT,
viewed:INT,
language:STRING,
referrer:STRING
>,
original struct<c_ip: STRING>
)
ROW FORMAT SERDE 'org.openx.data.jsonserde.JsonSerDe'
LOCATION 's3://gdbible-analytics/Converted'

99
terraform/lambda.tf Normal file
View File

@ -0,0 +1,99 @@
# -----------------------------------------------------------------------------------------------------------
# IAM Role for Redirect Lambda
resource "aws_iam_role" "lambda_redirect" {
name = "${var.site}-lambda-redirect-role"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": [
"edgelambda.amazonaws.com",
"lambda.amazonaws.com"
]
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
tags = {
Site = var.site
}
}
# -----------------------------------------------------------------------------------------------------------
# IAM Role for Log Parsing Lambda
resource "aws_iam_role" "lambda" {
name = "${var.site}-lambda-role"
assume_role_policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": "sts:AssumeRole",
"Principal": {
"Service": [
"edgelambda.amazonaws.com",
"lambda.amazonaws.com"
]
},
"Effect": "Allow",
"Sid": ""
}
]
}
EOF
tags = {
Site = var.site
}
}
resource "aws_iam_role_policy" "lambda" {
name = "${var.site}-lambda-execution-policy"
role = aws_iam_role.lambda.id
policy = <<EOF
{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"logs:CreateLogStream",
"logs:PutLogEvents",
"logs:CreateLogGroup"
],
"Effect": "Allow",
"Resource": "*"
},
{
"Effect": "Allow",
"Action": [
"s3:*"
],
"Resource": "arn:aws:s3:::*"
},
{
"Sid": "Invoke",
"Effect": "Allow",
"Action": [
"lambda:InvokeFunction"
],
"Resource": "arn:aws:lambda:*"
}
]
}
EOF
}

81
terraform/logging.tf Normal file
View File

@ -0,0 +1,81 @@
# -----------------------------------------------------------------------------------------------------------
# Grant the log parsing lambda access to the logs bucket
resource "aws_lambda_permission" "allow_bucket" {
statement_id = "AllowExecutionFromS3Bucket"
action = "lambda:InvokeFunction"
function_name = aws_lambda_function.logs_parser.arn
principal = "s3.amazonaws.com"
source_arn = aws_s3_bucket.logs.arn
}
# -----------------------------------------------------------------------------------------------------------
# Log Parsing Lambda
data "archive_file" "logs_parser" {
type = "zip"
source_dir = "${path.module}/files/decorate"
output_path = "${path.module}/files/decorate.zip"
}
resource "aws_lambda_function" "logs_parser" {
filename = data.archive_file.logs_parser.output_path
function_name = "${var.site}-lambda"
handler = "index.handler"
source_code_hash = data.archive_file.logs_parser.output_base64sha256
runtime = "nodejs12.x"
memory_size = "128"
timeout = "5"
role = aws_iam_role.lambda.arn
tags = {
Name = "${var.site}-log-dist"
Site = var.site
}
}
resource "aws_s3_bucket_notification" "bucket_notification" {
bucket = aws_s3_bucket.logs.id
lambda_function {
lambda_function_arn = aws_lambda_function.logs_parser.arn
events = ["s3:ObjectCreated:*"]
filter_prefix = "RAW/"
filter_suffix = ".gz"
}
}
# Reduce log retention to two weeks
resource "aws_cloudwatch_log_group" "logs_parser" {
name = "/aws/lambda/${aws_lambda_function.logs_parser.function_name}"
retention_in_days = 14
}
# -----------------------------------------------------------------------------------------------------------
# Athena Configuration
resource "aws_s3_bucket" "athena" {
bucket = "${var.site}-athena"
acl = "private"
tags = {
Name = "${var.site}-athena"
Site = var.site
}
}
resource "aws_athena_workgroup" "wg" {
name = "${var.site}-wg"
tags = {
Name = "${var.site}-wg"
Site = var.site
}
}
resource "aws_athena_database" "db" {
name = var.site
bucket = aws_s3_bucket.athena.id
}

122
terraform/src.tf Normal file
View File

@ -0,0 +1,122 @@
# -----------------------------------------------------------------------------------------------------------
# IAM User for Uploading
resource "aws_iam_user" "s3" {
name = "${var.site}-s3"
path = "/${var.site}/"
tags = {
Site = var.site
Category = "S3"
}
}
# resource "aws_iam_user_policy" "s3" {
# name = "test"
# user = "${aws_iam_user.s3.name}"
# policy = <<EOF
# {
# "Version": "2012-10-17",
# "Statement": [
# {
# "Effect": "Allow",
# "Action": "s3:*",
# "Resource": "*"
# }
# ]
# }
# EOF
# }
# This writes the s3 access key and secret to the terraform state file
resource "aws_iam_access_key" "s3" {
user = aws_iam_user.s3.name
}
# output s3_access {
# description = "S3 Upload User AccessKey"
# value = "${aws_iam_access_key.s3.id}"
# }
# output s3_secret {
# description = "S3 Upload User Secret"
# value = "${aws_iam_access_key.s3.secret}"
# }
# -----------------------------------------------------------------------------------------------------------
# Site Source Code
resource "aws_s3_bucket" "src" {
bucket = var.domain
acl = "public-read"
website {
index_document = "index.html"
error_document = "404.html"
}
tags = {
Name = "Site Source"
Site = var.site
}
}
resource "aws_s3_bucket_policy" "src" {
bucket = aws_s3_bucket.src.bucket
policy = <<POLICY
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Principal": {
"AWS": "${aws_iam_user.s3.arn}"
},
"Action": "s3:ListBucket",
"Resource": "${aws_s3_bucket.src.arn}"
},
{
"Effect": "Allow",
"Principal": {
"AWS": "${aws_iam_user.s3.arn}"
},
"Action": [
"s3:PutObject",
"s3:PutObjectAcl",
"s3:GetObject",
"s3:GetObjectAcl",
"s3:DeleteObject",
"s3:ListMultipartUploadParts",
"s3:AbortMultipartUpload"
],
"Resource": "${aws_s3_bucket.src.arn}/*"
},
{
"Sid": "PublicReadGetObject",
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": "${aws_s3_bucket.src.arn}/*"
}
]
}
POLICY
}
# resource "aws_s3_bucket" "redirect" {
# bucket = "www.${var.domain}"
# acl = "public-read"
# website {
# redirect_all_requests_to = var.domain
# }
# tags = {
# Name = "Redirect"
# Site = var.site
# }
# }

117
terraform/tracking.tf Normal file
View File

@ -0,0 +1,117 @@
# -----------------------------------------------------------------------------------------------------------
# Bucket for holding the tracking pixel file
resource "aws_s3_bucket" "pixel" {
bucket = "t.${var.domain}"
acl = "public-read"
cors_rule {
allowed_headers = ["*"]
allowed_methods = ["GET", "HEAD"]
allowed_origins = ["*"]
expose_headers = ["ETag"]
max_age_seconds = 3000
}
tags = {
Name = "Tracking Pixel"
Site = var.site
}
}
resource "aws_s3_bucket_object" "ipixel" {
bucket = aws_s3_bucket.pixel.bucket
key = "i"
source = "${path.module}/files/i.gif"
etag = filemd5("${path.module}/files/i.gif")
acl = "public-read"
content_type = "image/gif"
}
resource "aws_s3_bucket" "logs" {
bucket = "${var.site}-analytics"
tags = {
Name = "Logs Storage"
Site = var.site
}
}
# -----------------------------------------------------------------------------------------------------------
# Cloudfront Configuration for the tracking pixel
resource "aws_cloudfront_distribution" "tracking" {
origin {
domain_name = aws_s3_bucket.pixel.bucket_regional_domain_name
origin_id = "S3-${aws_s3_bucket.pixel.bucket}"
}
enabled = true
is_ipv6_enabled = true
comment = "Cloudfront distribution for tracking pixel"
logging_config {
include_cookies = true
bucket = aws_s3_bucket.logs.bucket_regional_domain_name
prefix = "RAW"
}
aliases = [
"t.${var.domain}"
]
default_cache_behavior {
allowed_methods = ["GET", "HEAD", "OPTIONS"]
cached_methods = ["GET", "HEAD"]
target_origin_id = "S3-${aws_s3_bucket.pixel.bucket}"
forwarded_values {
query_string = true
cookies {
forward = "all"
}
headers = [
"Origin",
"Access-Control-Request-Headers",
"Access-Control-Request-Method",
]
}
viewer_protocol_policy = "allow-all"
min_ttl = 0
default_ttl = 3600
max_ttl = 86400
}
restrictions {
geo_restriction {
restriction_type = "none"
}
}
viewer_certificate {
acm_certificate_arn = aws_acm_certificate.cert.arn
ssl_support_method = "sni-only"
}
tags = {
Name = "Tracking Site"
Site = var.site
}
}
resource "aws_route53_record" "tracking" {
name = "t.${var.domain}"
zone_id = aws_route53_zone.zone.zone_id
type = "A"
alias {
name = aws_cloudfront_distribution.tracking.domain_name
zone_id = aws_cloudfront_distribution.tracking.hosted_zone_id
evaluate_target_health = false
}
}

25
terraform/vars.tf Normal file
View File

@ -0,0 +1,25 @@
variable "site" {
type = string
description = "The name of the site"
default = "gdbible"
}
variable "domain" {
type = string
description = "The base domain name of the site that all these belong to."
default = "genderdysphoria.fyi"
}
variable "subdomains" {
type = list
default = [
"www",
"t"
]
}
provider "aws" {
profile = "default"
region = "us-east-1"
}

151
terraform/www.tf Normal file
View File

@ -0,0 +1,151 @@
resource "aws_s3_bucket" "temp_redirect" {
bucket = "${var.site}-redirect"
acl = "public-read"
website {
index_document = "index.html"
error_document = "index.html"
# redirect_all_requests_to = "https://curvyandtrans.com/p/740D5B/gender-dysphoria/"
}
}
resource "aws_s3_bucket_object" "redirect_page" {
bucket = aws_s3_bucket.temp_redirect.bucket
key = "index.html"
acl = "public-read"
content_type = "text/html"
metadata = {
"website-redirect-location" = "https://curvyandtrans.com/p/740D5B/gender-dysphoria/"
}
content = <<EOF
<html xmlns="http://www.w3.org/1999/xhtml"><head>
<meta http-equiv="refresh" content="0;URL='https://curvyandtrans.com/p/740D5B/gender-dysphoria/'" />
</head><body></body></html>
EOF
}
# -----------------------------------------------------------------------------------------------------------
# Cloudfront Configuration
resource "aws_cloudfront_distribution" "site" {
origin {
domain_name = aws_s3_bucket.temp_redirect.bucket_regional_domain_name
origin_id = "S3-Website-${aws_s3_bucket.src.website_endpoint}"
custom_origin_config {
origin_protocol_policy = "http-only"
http_port = "80"
https_port = "443"
origin_ssl_protocols = ["SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"]
}
}
enabled = true
is_ipv6_enabled = true
default_root_object = "index.html"
aliases = [
var.domain,
"www.${var.domain}"
]
default_cache_behavior {
allowed_methods = ["GET", "HEAD"]
cached_methods = ["GET", "HEAD"]
target_origin_id = "S3-Website-${aws_s3_bucket.src.website_endpoint}"
forwarded_values {
query_string = false
cookies {
forward = "none"
}
}
# lambda_function_association {
# event_type = "origin-request"
# lambda_arn = aws_lambda_function.index_redirect.qualified_arn
# include_body = false
# }
viewer_protocol_policy = "redirect-to-https"
min_ttl = 0
default_ttl = 86400
max_ttl = 31536000
}
restrictions {
geo_restriction {
restriction_type = "none"
}
}
viewer_certificate {
acm_certificate_arn = aws_acm_certificate.cert.arn
ssl_support_method = "sni-only"
minimum_protocol_version = "TLSv1.1_2016"
}
# viewer_certificate {
# cloudfront_default_certificate = true
# }
tags = {
Name = "Main Site"
Site = var.site
}
}
# -----------------------------------------------------------------------------------------------------------
# Domains
resource "aws_route53_record" "site" {
name = var.domain
zone_id = aws_route53_zone.zone.zone_id
type = "A"
alias {
name = aws_cloudfront_distribution.site.domain_name
zone_id = aws_cloudfront_distribution.site.hosted_zone_id
evaluate_target_health = false
}
}
resource "aws_route53_record" "www" {
name = "www.${var.domain}"
zone_id = aws_route53_zone.zone.zone_id
type = "A"
alias {
name = aws_cloudfront_distribution.site.domain_name
zone_id = aws_cloudfront_distribution.site.hosted_zone_id
evaluate_target_health = false
}
}
# -----------------------------------------------------------------------------------------------------------
# Lambda Subdirectory index.html Redirect
# data "archive_file" "index_redirect" {
# type = "zip"
# output_path = "${path.module}/files/index_redirect.js.zip"
# source_file = "${path.module}/files/index_redirect.js"
# }
# resource "aws_lambda_function" "index_redirect" {
# description = "index.html subdirectory redirect"
# filename = "${path.module}/files/index_redirect.js.zip"
# function_name = "folder-index-redirect"
# handler = "index_redirect.handler"
# source_code_hash = data.archive_file.index_redirect.output_base64sha256
# publish = true
# role = aws_iam_role.lambda_redirect.arn
# runtime = "nodejs12.x"
# tags = {
# Name = "${var.site}-index-redirect"
# Site = var.site
# }
# }