: Does Googlebot execute Google Tag Manager? I wanted to understand how Googlebot (and other crawlers) crawl my site. Specifically whether it passes a document.referrer and if it maintains localStorage

Posted in: #Googlebot #GoogleTagManager #Gtm #Javascript #Seo

I wanted to understand how Googlebot (and other crawlers) crawl my site. Specifically whether it passes a document.referrer and if it maintains localStorage keys, so I implemented a script via Google Tag Manager that detects these crawlers and logs data to Logstash.

This is the condition I'm using to detect crawler user agents (returns true for crawlers):

function() {
if(navigator.userAgent.indexOf('robot de Google') < 0 &&
navigator.userAgent.indexOf('Googlebot') < 0 &&
navigator.userAgent.indexOf('bingbot') < 0 &&
navigator.userAgent.indexOf('msnbot') < 0 &&
navigator.userAgent.indexOf('BingPreview') < 0 &&
navigator.userAgent.indexOf('Yahoo! Slurp') < 0) {
return false;
} else {
return true;
}
}

And this is the tag that sends sends a request to Logstash via an image pixel on the GTM Pageview event:

<script type="text/javascript">
(function (d) {

var pagePath = encodeURIComponent(document.location.pathname);
var pageReferrer = encodeURIComponent(document.referrer) || "null";
var userAgent = encodeURIComponent(navigator.userAgent);

var viewCount = Number(localStorage.getItem("preview_view_count")) + 1 || 1;
localStorage.setItem("preview_view_count", viewCount);

var js;
js = d.createElement('img');
js.style = 'display:none;';
js.alt = 'tracking img';
js.src = 'http://MY_LOGSTASH_ENDPOINT_DOMAIN/pixel.gif?EVENT=LogCrawl&USER_AGENT=' + userAgent + '&PAGE_PATH=' + pagePath + '&PAGE_REFERRER=' + pageReferrer + '&VIEW_COUNT=' + viewCount;
d.body.appendChild(js);
})(window.document);
</script>

Now when I look at Logstash, I only see 40 hits in the last 4 days from Googlebot, but Search Console reports ~50,000 pages crawled per day.

Has anyone tried to log Googlebot with GTM before? I'm trying to figure out if there is something wrong with my script, or if Googlebot just doesn't execute Javascript most of the time.

Any ideas are highly appreciated. Thanks.