twitterbot | A Python framework for creating interactive Twitter bots | Bot library

/news/detail.html?id=1 -> /api/v1/informations/ogp/1?lang=ja
/news/detail.html?id=1&lang=en -> /api/v1/informations/ogp/1?lang=en

/sport/detail.html?id=1 -> /api/v1/sports/ogp/1?lang=ja
/sport/detail.html?id=1&lang=en -> /api/v1/sports/ogp/1?lang=en

/event/common/detail.html?id=1 -> /api/v1/events/ogp/1?lang=ja
/event/common/detail.html?id=1 -> /api/v1/events/ogp/1?lang=ja
/event/special/detail.html?id=2&lang=en -> /api/v1/events/ogp/2?lang=en
/event/special/detail.html?id=2 -> /api/v1/events/ogp/2?lang=ja

    SetEnvIfNoCase User-Agent "^facebookexternalhit.*$" UA_FACEBOOK=1
    SetEnvIfNoCase User-Agent "^facebookplatform.*$" UA_FACEBOOK=1
    SetEnvIfNoCase User-Agent "^Twitterbot.*$" UA_TWITTER=1



    RewriteEngine on
    RewriteBase /

    RewriteCond %{ENV:UA_FACEBOOK} ^1$ [OR]
    RewriteCond %{ENV:UA_TWITTER} ^1$
    RewriteCond %{QUERY_STRING} (^|&)id=(\d+)&lang=(\w+)($|&)
    RewriteRule ^news/detail.html$ /api/v1/informations/ogp/%2?lang=%3 [R,L]
    RewriteRule ^sport/detail.html$ /api/v1/sports/ogp/%2?lang=%3 [R,L]
    RewriteRule ^event/common/detail.html$ /api/v1/events/ogp/%2?lang=%3 [R,L]
    RewriteRule ^event/special/detail.html$ /api/v1/events/ogp/%2?lang=%3 [R,L]

    RewriteCond %{ENV:UA_FACEBOOK} ^1$ [OR]
    RewriteCond %{ENV:UA_TWITTER} ^1$
    RewriteCond %{QUERY_STRING} (^|&)id=(\d+)($|&)
    RewriteRule ^news/detail.html$ /api/v1/informations/ogp/%2?lang=ja [R,L]
    RewriteRule ^sport/detail.html$ /api/v1/sports/ogp/%2?lang=ja [R,L]
    RewriteRule ^event/common/detail.html$ /api/v1/events/ogp/%2?lang=ja [R,L]
    RewriteRule ^event/special/detail.html$ /api/v1/events/ogp/%2?lang=ja [R,L]

RewriteEngine On
# If requested resource exists as a file or directory
# (REQUEST_FILENAME is only relative in virtualhost context, so not usable)
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_URI} -f [OR]
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_URI} -d
# Go to it as is
RewriteRule ^ - [L]

# If non existent
# If path ends with / and is not just a single /, redirect to without the trailing /
RewriteCond %{REQUEST_URI} ^.*/$
RewriteCond %{REQUEST_URI} !^/$
RewriteRule ^(.*)/$ $1 [R,QSA,L]      

# Handle Prerender.io
RequestHeader set X-Prerender-Token "notprovidingthiscode"

RewriteCond %{HTTP_USER_AGENT} baiduspider|facebookexternalhit|twitterbot|rogerbot|linkedinbot|embedly|quora\ link\ preview|showyoubot|outbrain|pinterest [NC,OR]
RewriteCond %{QUERY_STRING} _escaped_fragment_

# Proxy the request
RewriteRule ^(.*)$ http://service.prerender.io/https://%{HTTP_HOST}/$1 [P,L]

# If non existent
# Accept everything on index.html
RewriteRule ^ /index.html

❯ curl -vv localhost:8080
*   Trying 127.0.0.1...
* TCP_NODELAY set
* Connected to localhost (127.0.0.1) port 8080 (#0)
> GET / HTTP/1.1
> Host: localhost:8080
> User-Agent: curl/7.64.1
> Accept: */*
>
* Empty reply from server
* Connection #0 to host localhost left intact
curl: (52) Empty reply from server
* Closing connection 0

worker_processes 1;
daemon off;

events {
    worker_connections 1024;
}

http {
    root /var/www/html;

    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';

    access_log /dev/stdout main;
    error_log /dev/stderr error;

    sendfile on;
    tcp_nopush on;
    tcp_nodelay on;
    #keepalive low (5seconds), should force hackers to re-connect.
    keepalive_timeout  5;
    fastcgi_intercept_errors on;
    fastcgi_buffers 16 16k; 
    fastcgi_buffer_size 32k;
    default_type       application/octet-stream;

    #php max upload limit cannot be larger than this
    client_max_body_size 40m;

    gzip on;
    gzip_disable "msie6";
    gzip_min_length 256;
    gzip_comp_level 4;
    gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript application/javascript image/svg+xml;

    limit_req_zone $remote_addr zone=loginauth:10m rate=15r/s;

    include  /etc/nginx/mime.types;
    include  /etc/nginx/nginx-server.conf;
}

server {
    listen 8080 default_server;
    server_name "localhost";

    access_log /dev/stdout main;
    error_log /dev/stdout error;

    # pass the PHP scripts to FastCGI
    location ~ \.php$ {
        include        fastcgi_params;
        fastcgi_pass   unix:/home/www-data/php-fpm.sock;
        fastcgi_index  index.php;
        fastcgi_param  DOCUMENT_ROOT $realpath_root;
        fastcgi_param  SCRIPT_FILENAME $document_root$fastcgi_script_name;
        fastcgi_intercept_errors on;
    }

    #Deny access to .htaccess, .htpasswd...
    location ~ /\.ht {
        deny  all;
    }


    location ~* .(jpg|jpeg|png|gif|ico|css|js|pdf|doc|docx|odt|rtf|ppt|pptx|xls|xlsx|txt)$ {
      expires max;
    }

    location = /favicon.ico {
        log_not_found off;
        access_log off;
    }

    location = /robots.txt {
        allow all;
        log_not_found off;
        access_log off;
    }

    #Block bad-bots
    if ($http_user_agent ~* (360Spider|80legs.com|Abonti|AcoonBot|Acunetix|adbeat_bot|AddThis.com|adidxbot|ADmantX|AhrefsBot|AngloINFO|Antelope|Applebot|BaiduSpider|BeetleBot|billigerbot|binlar|bitlybot|BlackWidow|BLP_bbot|BoardReader|Bolt\ 0|BOT\ for\ JCE|Bot\ mailto\:craftbot@yahoo\.com|casper|CazoodleBot|CCBot|checkprivacy|ChinaClaw|chromeframe|Clerkbot|Cliqzbot|clshttp|CommonCrawler|comodo|CPython|crawler4j|Crawlera|CRAZYWEBCRAWLER|Curious|Curl|Custo|CWS_proxy|Default\ Browser\ 0|diavol|DigExt|Digincore|DIIbot|discobot|DISCo|DoCoMo|DotBot|Download\ Demon|DTS.Agent|EasouSpider|eCatch|ecxi|EirGrabber|Elmer|EmailCollector|EmailSiphon|EmailWolf|Exabot|ExaleadCloudView|ExpertSearchSpider|ExpertSearch|Express\ WebPictures|ExtractorPro|extract|EyeNetIE|Ezooms|F2S|FastSeek|feedfinder|FeedlyBot|FHscan|finbot|Flamingo_SearchEngine|FlappyBot|FlashGet|flicky|Flipboard|g00g1e|Genieo|genieo|GetRight|GetWeb\!|GigablastOpenSource|GozaikBot|Go\!Zilla|Go\-Ahead\-Got\-It|GrabNet|grab|Grafula|GrapeshotCrawler|GTB5|GT\:\:WWW|Guzzle|harvest|heritrix|HMView|HomePageBot|HTTP\:\:Lite|HTTrack|HubSpot|ia_archiver|icarus6|IDBot|id\-search|IlseBot|Image\ Stripper|Image\ Sucker|Indigonet|Indy\ Library|integromedb|InterGET|InternetSeer\.com|Internet\ Ninja|IRLbot|ISC\ Systems\ iRc\ Search\ 2\.1|jakarta|Java|JetCar|JobdiggerSpider|JOC\ Web\ Spider|Jooblebot|kanagawa|KINGSpider|kmccrew|larbin|LeechFTP|libwww|Lingewoud|LinkChecker|linkdexbot|LinksCrawler|LinksManager\.com_bot|linkwalker|LinqiaRSSBot|LivelapBot|ltx71|LubbersBot|lwp\-trivial|Mail.RU_Bot|masscan|Mass\ Downloader|maverick|Maxthon$|Mediatoolkitbot|MegaIndex|MegaIndex|megaindex|MFC_Tear_Sample|Microsoft\ URL\ Control|microsoft\.url|MIDown\ tool|miner|Missigua\ Locator|Mister\ PiX|mj12bot|Mozilla.*Indy|Mozilla.*NEWT|MSFrontPage|msnbot|Navroad|NearSite|NetAnts|netEstate|NetSpider|NetZIP|Net\ Vampire|NextGenSearchBot|nutch|Octopus|Offline\ Explorer|Offline\ Navigator|OpenindexSpider|OpenWebSpider|OrangeBot|Owlin|PageGrabber|PagesInventory|panopta|panscient\.com|Papa\ Foto|pavuk|pcBrowser|PECL\:\:HTTP|PeoplePal|Photon|PHPCrawl|planetwork|PleaseCrawl|PNAMAIN.EXE|PodcastPartyBot|prijsbest|proximic|psbot|purebot|pycurl|QuerySeekerSpider|R6_CommentReader|R6_FeedFetcher|RealDownload|ReGet|Riddler|Rippers\ 0|rogerbot|RSSingBot|rv\:1.9.1|RyzeCrawler|SafeSearch|SBIder|Scrapy|Scrapy|Screaming|SeaMonkey$|search.goo.ne.jp|SearchmetricsBot|search_robot|SemrushBot|Semrush|SentiBot|SEOkicks|SeznamBot|ShowyouBot|SightupBot|SISTRIX|sitecheck\.internetseer\.com|siteexplorer.info|SiteSnagger|skygrid|Slackbot|Slurp|SmartDownload|Snoopy|Sogou|Sosospider|spaumbot|Steeler|sucker|SuperBot|Superfeedr|SuperHTTP|SurdotlyBot|Surfbot|tAkeOut|Teleport\ Pro|TinEye-bot|TinEye|Toata\ dragostea\ mea\ pentru\ diavola|Toplistbot|trendictionbot|TurnitinBot|turnit|Twitterbot|URI\:\:Fetch|urllib|Vagabondo|Vagabondo|vikspider|VoidEYE|VoilaBot|WBSearchBot|webalta|WebAuto|WebBandit|WebCollage|WebCopier|WebFetch|WebGo\ IS|WebLeacher|WebReaper|WebSauger|Website\ eXtractor|Website\ Quester|WebStripper|WebWhacker|WebZIP|Web\ Image\ Collector|Web\ Sucker|Wells\ Search\ II|WEP\ Search|WeSEE|Wget|Widow|WinInet|woobot|woopingbot|worldwebheritage.org|Wotbox|WPScan|WWWOFFLE|WWW\-Mechanize|Xaldon\ WebSpider|XoviBot|yacybot|Yahoo|YandexBot|Yandex|YisouSpider|zermelo|Zeus|zh-CN|ZmEu|ZumBot|ZyBorg) ) {
              return 444;
    }

    include /etc/nginx/nginx-locations.conf;
    include /var/www/nginx/locations/*;
}

# Deny all attempts to access hidden files such as .htaccess, .htpasswd, .DS_Store (Mac).
# Keep logging the requests to parse later (or to pass to firewall utilities such as fail2ban)
location ~ /\. {
    deny all;
}

# Deny access to any files with a .php extension in the uploads directory for the single site
location ~ ^/wp-content/uploads/.*\.php$ {
    deny all;
}

#Deny access to wp-content folders for suspicious files
location ~* ^/(wp-content)/(.*?)\.(zip|gz|tar|bzip2|7z)\$ { deny all; }
location ~ ^/wp-content/uploads/sucuri { deny all; }
location ~ ^/wp-content/updraft { deny all; }
location ~* ^/wp-content/uploads/.*.(html|htm|shtml|php|js|swf)$ {
        deny all;
}

# Block PHP files in includes directory.
location ~* /wp-includes/.*\.php\$ {
  deny all;
}

# Deny access to any files with a .php extension in the uploads directory
# Works in sub-directory installs and also in multisite network
# Keep logging the requests to parse later (or to pass to firewall utilities such as fail2ban)
location ~* /(?:uploads|files|wp-content|wp-includes)/.*\.php$ {
    deny all;
}

# Block nginx-help log from public viewing
location ~* /wp-content/uploads/nginx-helper/ { deny all; }

# Deny access to any files with a .php extension in the uploads directory
# Works in sub-directory installs and also in multisite network
location ~* /(?:uploads|files)/.*\.php\$ { deny all; }

# Deny access to uploads that aren’t images, videos, music, etc.
location ~* ^/wp-content/uploads/.*.(html|htm|shtml|php|js|swf|css)$ {
    deny all;
}



location / {
        # This is cool because no php is touched for static content.
        # include the "?$args" part so non-default permalinks doesn't break when using query string
                index  index.php index.html;
        try_files $uri $uri/ /index.php?$args;
}

# More ideas from:
# https://gist.github.com/ethanpil/1bfd01a817a8198369efec5c4cde6628

location ~* /(\.|wp-config\.php|wp-config\.txt|changelog\.txt|readme\.txt|readme\.html|license\.txt) { deny all; }

# Make sure files with the following extensions do not get loaded by nginx because nginx would display the source code, and these files can contain PASSWORDS!
location ~* \.(engine|inc|info|install|make|module|profile|test|po|sh|.*sql|theme|tpl(\.php)?|xtmpl)\$|^(\..*|Entries.*|Repository|Root|Tag|Template)\$|\.php_
{
    return 444;
}
#nocgi
location ~* \.(pl|cgi|py|sh|lua)\$ {
    return 444;
}
#disallow
location ~* (w00tw00t) {
    return 444;
}

'use strict';
 
function buildReleaseResponse( request ) {
     
    const content = `<\!DOCTYPE html>
        
            
                
                
               
                
                
                
                
                
               
                Hello, World
            
            
                Open Graph Test
            
        `;
   
    return {
        statusCode: 200,
        statusDescription: 'OK',
        headers: {
            "content-type": [
                {
                    "key": "Content-Type",
                    "value": "text/html; charset=utf-8"
                }
            ]
        },
        body: content.toString()
    };
   
}
 

exports.handler = ( event, context, callback ) => {
    const { request, response } = event.Records[0].cf;
                                           
    let userAgentStr = "";
       
    if (request.headers['user-agent']) {
      if (request.headers['user-agent'].length > 0) {
          userAgentStr = request.headers['user-agent'][0].value;
      }
    }
   
    let newResponse = null;
   
    if ( userAgentStr.match(/facebookexternalhit|twitterbot/i) ) {
        if ( request.uri.startsWith("/radio/release/") ) {
            newResponse = buildReleaseResponse(request);
        }
    }
   
    if ( newResponse === null ) {
        console.log("Passthrough.");
        callback(null, request);
    }
    else {
        console.log("Overriding response with: " + JSON.stringify(newResponse));
        callback(null, newResponse);
    }   
};

{
    "statusCode": 200,
    "statusDescription": "OK",
    "headers": {
        "content-type": [{
            "key": "Content-Type",
            "value": "text/html; charset=utf-8"
        }],
        "cache-control": [{
            "key": "Cache-Control",
            "value": "max-age=100"
        }]
    },
    "body": "\n            \n                \n                    \n                    \n                    \n                    \n                    \n                    \n                    \n                    \n                    \n                    Kompoz.com\n                \n                \n                    Yo Dog\n                \n            "
}

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.chrome.options import Options
import time

def account_info():
    with open('account_info.txt', 'r') as f:
        info = f.read().split()
        email = info[0]
        password = [1]
        return email, password

email, password = account_info()

options = Options()
options.add_argument("start.maximized")
driver = webdriver.Chrome(options=options)



driver.get("https://twitter.com/i/flow/login")

time.sleep(1)

email_xpath = '//*[@id="layers"]/div[2]/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[1]/div/div[2]/label/div/div[2]/div/input'
next_xpath = '//*[@id="layers"]/div[2]/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[2]/div/div'
password_xpath = '//*[@id="layers"]/div[2]/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[1]/div/div[2]/div/label/div/div[2]/div/input'
login_xpath = '//*[@id="layers"]/div[2]/div/div/div/div/div/div[2]/div[2]/div/div/div[2]/div[2]/div[2]/div/div'


time.sleep(1)

driver.find_element_by_xpath(email_xpath).send_keys(email)
time.sleep(0.5)
driver.find_element_by_xpath(next_xpath).click()
time.sleep(0.5)
driver.find_element_by_xpath(password_xpath).send_keys(password)
time.sleep(0.5)
driver.find_element_by_xpath(login_xpath).click()

server {
    listen 80 default_server;
    root /usr/share/nginx/html;
 
    index  index.html;

    location / {
        try_files $uri @prerender;
    }
 
    location @prerender {
        proxy_set_header X-Prerender-Token TOKEN_HERE;
        
        set $prerender 0;
        if ($http_user_agent ~* "googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest\/0\.|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp") {
            set $prerender 1;
        }
        if ($args ~ "_escaped_fragment_") {
            set $prerender 1;
        }
        if ($http_user_agent ~ "Prerender") {
            set $prerender 0;
        }
        if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
            set $prerender 0;
        }
        
        #resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
        resolver 8.8.8.8;
        resolver_timeout 60s;
 
        if ($prerender = 1) {
            
            #setting prerender as a variable forces DNS resolution since nginx caches IPs and doesnt play well with load balancing
            set $prerender "service.prerender.io";
            rewrite .* /$scheme://$host$request_uri? break;
            proxy_pass http://$prerender;
        }
        if ($prerender = 0) {
            rewrite .* /index.html break;
        }
    }
}

FROM node:12.7-alpine AS build
ARG environment=production

WORKDIR /usr/src/app
COPY package.json package-lock.json ./
RUN npm install
COPY . .
RUN npm run build -- --configuration $environment
# Two stage build because we do not need node-related things
FROM nginx:1.17.1-alpine

RUN apk add --no-cache curl
COPY --from=build /usr/src/app/dist/app /usr/share/nginx/html
COPY prerender-nginx.conf /etc/nginx/conf.d/default.conf

EXPOSE 80

   server {
    listen       80;
    server_name  localhost;
    absolute_redirect off;
    root   /usr/share/nginx/html;
    index  index.html index.htm;
    #charset koi8-r;
    #access_log  /var/log/nginx/host.access.log  main;

    location / {
        try_files $uri @prerender;
        #try_files $uri $uri/ /index.html; # force routes to index.html
    }

    #error_page  404              /404.html;
    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

        gzip on;
        gzip_vary on;
        gzip_min_length 10240;
        gzip_proxied expired no-cache no-store private auth;
        gzip_types text/plain text/css text/xml text/javascript application/x-javascript application/xml;

    location @prerender {
        proxy_set_header X-Prerender-Token JEWpuxaXuzdqvd7tKD1l;

        set $prerender 0;
        if ($http_user_agent ~* "googlebot|bingbot|yandex|baiduspider|twitterbot|facebookexternalhit|rogerbot|linkedinbot|embedly|quora link preview|showyoubot|outbrain|pinterest\/0\.|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp") {
            set $prerender 1;
        }
        if ($args ~ "_escaped_fragment_") {
            set $prerender 1;
        }
        if ($http_user_agent ~ "Prerender") {
            set $prerender 0;
        }
        if ($uri ~* "\.(js|css|xml|less|png|jpg|jpeg|gif|pdf|doc|txt|ico|rss|zip|mp3|rar|exe|wmv|doc|avi|ppt|mpg|mpeg|tif|wav|mov|psd|ai|xls|mp4|m4a|swf|dat|dmg|iso|flv|m4v|torrent|ttf|woff|svg|eot)") {
            set $prerender 0;
        }

        #resolve using Google's DNS server to force DNS resolution and prevent caching of IPs
        resolver 8.8.8.8;

        if ($prerender = 1) {
            #setting prerender as a variable forces DNS resolution since nginx caches IPs and doesnt play well with load balancing
            set $prerender "service.prerender.io";
            # rewrite .* /$scheme://$host$request_uri? break;
            # Following domain need to be dynamic
            rewrite .* /https://www.drivemate.asia$request_uri? break;
            proxy_pass http://$prerender;
        }
        if ($prerender = 0) {
            #try_files $uri $uri/ /index.html; # force routes to index.html
            rewrite .* /index.html break;
        }
    }

Options +FollowSymLinks
RewriteEngine On

RewriteCond %{HTTP_HOST} ^www\.(.*)$ [NC]
RewriteRule ^(.*)$ https://%1/$1 [R=301,L]

RewriteCond %{HTTPS} !on
RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI} [R=301,L]


    RequestHeader set X-Prerender-Token "TOKEN"
    RequestHeader set X-Prerender-Version "prerender-apache@2.0.0"



    RewriteEngine On
    
        RewriteCond %{HTTP_USER_AGENT} googlebot|bingbot|yandex|baiduspider|facebookexternalhit|twitterbot|rogerbot|linkedinbot|embedly|quora\ link\ preview|showyoubot|outbrain|pinterest\/0\.|pinterestbot|slackbot|vkShare|W3C_Validator|whatsapp [NC,OR]
        RewriteCond %{QUERY_STRING} _escaped_fragment_
        RewriteCond %{REQUEST_URI} ^(?!.*?(\.js|\.css|\.xml|\.less|\.png|\.jpg|\.jpeg|\.gif|\.pdf|\.doc|\.txt|\.ico|\.rss|\.zip|\.mp3|\.rar|\.exe|\.wmv|\.doc|\.avi|\.ppt|\.mpg|\.mpeg|\.tif|\.wav|\.mov|\.psd|\.ai|\.xls|\.mp4|\.m4a|\.swf|\.dat|\.dmg|\.iso|\.flv|\.m4v|\.torrent|\.ttf|\.woff|\.svg))

        RewriteRule ^(index\.html|index\.php)?(.*) https://service.prerender.io/%{REQUEST_SCHEME}://%{HTTP_HOST}/$2 [P,END]
    

- crawlers/
           - crawlers.php
           - share/
                   - 404.json
                   - about.json
                   - work.json
- .htaccess
- index.html

'.PHP_EOL;
    $html .= ''.PHP_EOL;

    $html .= ''.PHP_EOL;

    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;

    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;
    
    $html .= ''.PHP_EOL;

    $html .= ''.PHP_EOL;
    $html .= ''.PHP_EOL;

    $html .= '';

    echo $html;
}

?>

{
  "title": "404: Not Found | My Website",
  "description": "My awesome description.",
  "image": "https://www.mywebsite.com/assets/images/share/404.jpg",
  "url": "https://www.mywebsite.com",
}

RewriteEngine On
RewriteBase /

# Allow robots.txt to pass through
RewriteRule ^robots.txt - [L]

# Allow social media crawlers to work
RewriteCond %{HTTP_USER_AGENT} (facebookexternalhit/[0-9]|Twitterbot)
RewriteRule ^(.+)$ /crawlers/crawlers.php?page=$1 [NC,L]

# If an existing asset or directory is requested go to it as it is
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_URI} -f [OR]
RewriteCond %{DOCUMENT_ROOT}%{REQUEST_URI} -d
RewriteRule ^ - [L]

# If the requested resource doesn't exist use index.html
RewriteRule ^ /index.html