Crunchbase Data API v3.1 to Google Sheets - api

I'm trying to pull data from the Crunchbase Open Data Map to a Google Spreadsheet. I'm following Ben Collins's script but it no longer works since the upgrade from v3 to v3.1. Anyone had any luck modifying the script for success?
var USER_KEY = 'insert your API key in here';
// function to retrive organizations data
function getCrunchbaseOrgs() {
var ss = SpreadsheetApp.getActiveSpreadsheet();
var sheet = ss.getSheetByName('Organizations');
var query = sheet.getRange(3,2).getValue();
// URL and params for the Crunchbase API
var url = 'https://api.crunchbase.com/v/3/odm-organizations?query=' + encodeURI(query) + '&user_key=' + USER_KEY;
var json = getCrunchbaseData(url,query);
if (json[0] === "Error:") {
// deal with error with fetch operation
sheet.getRange(5,1,sheet.getLastRow(),2).clearContent();
sheet.getRange(6,1,1,2).setValues([json]);
}
else {
if (json[0] !== 200) {
// deal with error from api
sheet.getRange(5,1,sheet.getLastRow(),2).clearContent();
sheet.getRange(6,1,1,2).setValues([["Error, server returned code:",json[0]]]);
}
else {
// correct data comes back, filter down to match the name of the entity
var data = json[1].data.items.filter(function(item) {
return item.properties.name == query;
})[0].properties;
// parse into array for Google Sheet
var outputData = [
["Name",data.name],
["Homepage",data.homepage_url],
["Type",data.primary_role],
["Short description",data.short_description],
["Country",data.country_code],
["Region",data.region_name],
["City name",data.city_name],
["Blog url",data.blog_url],
["Facebook",data.facebook_url],
["Linkedin",data.linkedin_url],
["Twitter",data.twitter_url],
["Crunchbase URL","https://www.crunchbase.com/" + data.web_path]
];
// clear any old data
sheet.getRange(5,1,sheet.getLastRow(),2).clearContent();
// insert new data
sheet.getRange(6,1,12,2).setValues(outputData);
// add image with formula and format that row
sheet.getRange(5,2).setFormula('=image("' + data.profile_image_url + '",4,50,50)').setHorizontalAlignment("center");
sheet.setRowHeight(5,60);
}
}
}
This code no longer pulls data as expected.

I couldn't confirm about the error messages when you ran the script. So I would like to show about the clear difference point. It seems that the endpoint was changed from https://api.crunchbase.com/v/3/ to https://api.crunchbase.com/v3.1/. So how about this modification?
From :
var url = 'https://api.crunchbase.com/v/3/odm-organizations?query=' + encodeURI(query) + '&user_key=' + USER_KEY;
To :
var url = 'https://api.crunchbase.com/v3.1/odm-organizations?query=' + encodeURI(query) + '&user_key=' + USER_KEY;
Note :
From your script, I couldn't also find query. So if the script doesn't work even when you modified the endpoint, please confirm about it. You can see the detail of API v3 Compared to API v3.1 is here.
References :
API v3 Compared to API v3.1
Using the API
If this was not useful for you, I'm sorry.

Related

API call from google sheets is too long

I am trying to write a script in google sheets to update my 3commas bots. The API requires a number of mandatory fields are passed even when there's only 1 item that needs to be updated.
The code I have is below and it uses the values already read from the platform updating only the base_order_volume value. This works perfectly except for when the pairs value is long (more the 2k chars) and then I get an error from the UrlFetchApp call because the URL is too long.
var sheet = SpreadsheetApp.getActiveSheet();
var key = sheet.getRange('F4').getValue();
var secret = sheet.getRange('F5').getValue();
var baseUrl = "https://3commas.io";
var editBots = "/ver1/bots/"+bots[botCounter].id+"/update";
var patchEndPoint = "/public/api"+editBots+"?";
.
.
[loop around values in sheet]
.
.
var BaseOrder=Number(sheet.getRange(rowCounter,12).getValue().toFixed(2));
var botParams = {
"name": bots[botCounter].name,
"pairs": bots[botCounter].pairs,
"max_active_deals": bots[botCounter].max_active_deals,
"base_order_volume": BaseOrder,
"take_profit": Number(bots[botCounter].take_profit),
"safety_order_volume": bots[botCounter].safety_order_volume,
"martingale_volume_coefficient": bots[botCounter].martingale_volume_coefficient,
"martingale_step_coefficient": Number(bots[botCounter].martingale_step_coefficient),
"max_safety_orders": bots[botCounter].max_safety_orders,
"active_safety_orders_count": Number(bots[botCounter].active_safety_orders_count),
"safety_order_step_percentage": Number(bots[botCounter].safety_order_step_percentage),
"take_profit_type": bots[botCounter].take_profit_type,
"strategy_list": bots[botCounter].strategy_list,
"bot_id": bots[botCounter].id
};
var keys = Object.keys(botParams);
var totalParams = keys.reduce(function(q, e, i) {
q += e + "=" + encodeURIComponent(JSON.stringify(botParams[e])) + (i != keys.length - 1 ? "&" : "");
return q;
},endPoint);
var signature = Utilities.computeHmacSha256Signature(totalParams, secret);
signature = signature.map(function(e) {return ("0" + (e < 0 ? e + 256 : e).toString(16)).slice(-2)}).join("");
var headers = {
'APIKEY': key,
'Signature': signature,
};
try {
var params = {
'method': 'PATCH',
'headers': headers,
'muteHttpExceptions': true
};
var response = JSON.parse(UrlFetchApp.fetch(baseUrl + totalParams, params).getContentText());
I have tried to set the botParams as a payload in the params but when I do the signature is incorrect.
I anyone knows how to use sheets to make a call using extensive length of parameters I'd appreciate any help at all
Some sample data for the bots array would be
{
"name": "TestBot",
"base_order_volume": 0.001,
"take_profit": 1.5,
"safety_order_volume": 0.001,
"martingale_volume_coefficient": 2,
"martingale_step_coefficient": 1,
"max_safety_orders": 1,
"active_safety_orders_count": 1,
"safety_order_step_percentage": 2.5,
"take_profit_type": "total",
"stop_loss_percentage": 0,
"cooldown": 0,
"pairs": ["BTC_ADA","BTC_TRX"],
"trailing_enabled":"true",
"trailing_deviation":0.5,
"strategy_list": [{"strategy":"cqs_telegram"}]
}
Thanks in advance
I'd consider using a Cloud Function to either do the heavy lifting, or, if you're worried about costs, use it as a proxy. You can then call the cloud function from Google Sheets. Cloud Functions can be written in whatever language you're most comfortable with, including Node.
Check the GCP pricing calculator to see what the cost would be. For many cases it would be completely free.
This should give you a sense of how to use cloud functions for CSV creation:
https://codelabs.developers.google.com/codelabs/cloud-function2sheet#0
Here is a SO question with an answer that explains how to query cloud functions with authentication.

Community Connector getData() Request only uses the first two schema fields, not all four

I am building a Community Connector between Google Data Studio and SpyFu.com, in order to funnel SEO information for a specific url into the GDS Dashboard.
However, My getData() request only contains the first two fields from my Schema. As you can see, I have four listed in the code. The result is only the first two fields in the schema are printed to GDS.
I've been through tutorials, official documentation, YouTube videos, looked this issue up on google and checked out the community resources on GitHub.
//Step Two: Define getConfig()
function getConfig(request) {
var cc = DataStudioApp.createCommunityConnector();
var config = cc.getConfig();
config.newInfo()
.setId('instructions')
.setText('Give me SpyFu information on the following domain:');
config.newTextInput()
.setId('domain')
.setName('Enter the domain to search')
.setHelpText('e.g. ebay.com')
.setPlaceholder('ebay.com');
config.newTextInput()
.setId('SECRET_KEY')
.setName('Enter your API Secret Key')
.setHelpText('e.g. A1B2C3D4')
.setPlaceholder('A1B2C3D4');
config.setDateRangeRequired(false);
return config.build();
}
//Step Three: Define getSchema()
function getFields(request) {
var cc = DataStudioApp.createCommunityConnector();
var fields = cc.getFields();
var types = cc.FieldType;
var aggregations = cc.AggregationType;
fields.newDimension()
.setId('Keyword')
.setName('Keywords')
.setDescription('The keywords most often attributed to this domain.')
.setType(types.TEXT);
fields.newMetric()
.setId('Rank')
.setName('Rankings')
.setDescription('The ranking of the target site keyword on the Google Search Page.')
.setType(types.NUMBER);
fields.newMetric()
.setId('Local_Monthly_Searches')
.setName('Local Searches per Month')
.setDescription('Number of times, locally, that people have searched for this term within in the last month.')
.setType(types.NUMBER);
fields.newMetric()
.setId('Global_Monthly_Searches')
.setName('Global Searches per Month')
.setDescription('Number of times, globally, that people have searched for this term within in the last month.')
.setType(types.NUMBER);
return fields;
}
function getSchema(request) {
var fields = getFields(request).build();
return { schema: fields };
}
//Step Four: Define getData()
function responseToRows(requestedFields, response, domain) {
// Transform parsed data and filter for requested fields
return response.map(function(Array) {
var row = [];
requestedFields.asArray().forEach(function (field) {
switch (field.getId()) {
case 'Keyword':
return row.push(Array.term);
case 'Rank':
return row.push(Array.position);
case 'Local_Monthly_Searches':
return row.push(Array.exact_local_monthly_search_volume);
case 'Global_Monthly_Searches':
return row.push(Array.exact_global_monthly_search_volume);
case 'domain':
return row.push(domain);
default:
return row.push('');
}
});
return { values: row };
});
}
function getData(request) {
console.log("Request from Data Studio");
console.log(request);
var requestedFieldIds = request.fields.map(function(field) {
return field.name;
});
var requestedFields = getFields().forIds(requestedFieldIds);
// Fetch data from API
var url = [
'https://www.spyfu.com/apis/url_api/organic_kws?q='
+ request.configParams.domain
+ '&r=20'
+ '&p=[1 TO 10]'
+ '&api_key='
+ request.configParams.SECRET_KEY,
];
try {
var response = UrlFetchApp.fetch(url.join(''));
} catch (e) {
DataStudioApp.createCommunityConnector()
.newUserError()
.setDebugText('Failed URL Fetch Attempt. Exception details: ' + e)
.setText('There was an error accessing this domain. Try again later, or file an issue if this error persists.')
.throwException();
}
console.log("Response from API");
console.log(response);
//Parse data from the API
try {
var parsedResponse = JSON.parse(response);
} catch (e) {
DataStudioApp.createCommunityConnector()
.newUserError()
.setDebugText('Error parsing the JSON data. Exception details: ' + e)
.setText('There was an error parsing the JSON data. Try again later, or file an issue if this error persists.')
.throwException();
}
var rows = responseToRows(requestedFields, parsedResponse);
return {
schema: requestedFields.build(),
rows: rows
};
}
I need the GDS to post four columns of data. They are, "Keyword", "Rank", "Local Monthly Searches" and "Global Monthly searches".
I cannot figure out how to create a "fixed schema" so that the system always prints these four columns of data at every request. The tutorials and various documentation say it's possible, but not how to do it. Please help!
The number of metrics initially called up by the Google Community Connector is handled from the front-end, via Google Data Studio.
The back-end system (the Connector) only initially posts the default dimension and default metric. Getting the rest of the schemas to post should be handled when you are building a report on Google Data Studio. Simply click on the data set, select "data" on the right-hand menu, scroll down to either Metrics or Dimensions, and pick the ones you wish to add to the current set.
Note that these are the fields you established earlier in the coding process, when you were setting up your schemas.
Here, you're filtering your defined schema for fields that are present on the request object received by getData().
var requestedFieldIds = request.fields.map(function(field) {
return field.name;
});
var requestedFields = getFields().forIds(requestedFieldIds);
The visualization in Google Data Studio that is the catalyst for the request will determine which fields are requested.

Google API for getting maximum number of licenses in a Google Apps domain

I have a Google Apps Script function used for setting up accounts for new employees in our Google Apps domain.
The first thing it does is makes calls to the Google Admin Settings API and retrieves the currentNumberOfUsers and maximumNumberOfUsers, so it can see if there are available seats (otherwise a subsequent step where the user is created using the Admin SDK Directory API would fail).
It's been working fine until recently when our domain had to migrate from Postini to Google Vault for email archiving.
Before the migration, when creating a Google Apps user using the Admin SDK Directory API, it would increment the currentNumberOfUsers by 1 and the new user account user would automatically have access to all Google Apps services.
Now after the migration, when creating a Google Apps user, they aren't automatically assigned a "license," so I modified my script to use the Enterprise License Manager API and now it assigns a "Google-Apps-For-Business" license. That works fine.
However, the currentNumberOfUsers is now different from the number of assigned licenses, and "Google-Apps-For-Business" is only one of several different types of licenses available.
I can get the current number of assigned "Google-Apps-For-Business" licenses by running this:
var currentXml = AdminLicenseManager.LicenseAssignments.listForProductAndSku('Google-Apps', 'Google-Apps-For-Business', 'domain.com', {maxResults: 1000});
var current = currentXml.items.toString().match(/\/sku\/Google-Apps-For-Business\/user\//g).length;
But the number that produces is different from currentNumberOfUsers.
All I really need to do now is get the maximum number of owned "Google-Apps-For-Business" licenses so the new employee setup script can determine whether there are any available.
I checked the API Reference documentation for the following APIs but...
Enterprise License Manager API → Doesn't have a method for getting the maximum or available number of licenses.
Google Admin Settings API → Doesn't deal with licenses, only "users."
Admin SDK Directory API User resource → Doesn't deal with licenses.
Google Apps Reseller API → This API seems to have what I need, but it's only for Reseller accounts.
I know I can program my new employee setup script to just have a try/catch seeing if it would be able to create the user and assign the license, and end the script execution gracefully if it can't, but that doesn't seem efficient.
Also, part of the old script was that if there were less than X seats available, it would email me a heads-up to order more. I can program a loop that attempts to repeatedly create dummy users and assign them licenses and count the number of times it can do that before it fails, then delete all the dummy users, but, once again, that's not efficient at all.
Any ideas?
Update 3/11/2020: Since the Admin Settings API had shut down a few years ago I've been using the Enterprise License Manager API to get the current number of used licenses, like this:
function getCurrentNumberOfUsedGoogleLicenses(skuId) {
var success = false, error = null, count = 0;
var adminEmail = 'admin#domain.com';
var gSuiteDomain = adminEmail.split('#')[1];
// for more information on the domain-wide delegation:
// https://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority
// the getDomainWideDelegationService() function uses this:
// https://github.com/gsuitedevs/apps-script-oauth2
var service = getDomainWideDelegationService('EnterpriseLicenseManager: ', 'https://www.googleapis.com/auth/apps.licensing', adminEmail);
if (skuId == 'Google-Apps-Unlimited') var productId = 'Google-Apps';
else return { success: success, error: "Unsupported skuId", count: count };
var requestBody = {};
requestBody.headers = {'Authorization': 'Bearer ' + service.getAccessToken()};
requestBody.method = "GET";
requestBody.muteHttpExceptions = false;
var data, pageToken, pageTokenString;
var maxAttempts = 5;
var currentAttempts = 0;
var pauseBetweenAttemptsSeconds = 3;
loopThroughPages:
do {
if (typeof pageToken === 'undefined') pageTokenString = "";
else pageTokenString = "&pageToken=" + encodeURIComponent(pageToken);
var url = 'https://www.googleapis.com/apps/licensing/v1/product/' + productId + '/sku/' + skuId + '/users?maxResults=1000&customerId=' + gSuiteDomain + pageTokenString;
try {
currentAttempts++;
var response = UrlFetchApp.fetch(url, requestBody);
var result = JSON.parse(response.getContentText());
if (result.items) {
var licenseAssignments = result.items;
var licenseAssignmentsString = '';
for (var i = 0; i < licenseAssignments.length; i++) {
licenseAssignmentsString += JSON.stringify(licenseAssignments[i]);
}
if (skuId == 'Google-Apps-Unlimited') count += licenseAssignmentsString.match(/\/sku\/Google-Apps-Unlimited\/user\//g).length;
currentAttempts = 0; // reset currentAttempts before the next page
}
} catch(e) {
error = "Error: " + e.message;
if (currentAttempts >= maxAttempts) {
error = 'Exceeded ' + maxAttempts + ' attempts to get license count: ' + error;
break loopThroughPages;
}
} // end of try catch
if (result) pageToken = result.nextPageToken;
} while (pageToken);
if (!error) success = true;
return { success: success, error: error, count: count };
}
However, there still does not appear to be a way to get the maximum number available to the domain using this API.
Use CustomerUsageReports.
jay0lee is kind enough to provide the GAM source code in Python. I crudely modified the doGetCustomerInfo() function into Apps Script thusly:
function getNumberOfLicenses() {
var tryDate = new Date();
var dateString = tryDate.getFullYear().toString() + "-" + (tryDate.getMonth() + 1).toString() + "-" + tryDate.getDate().toString();
while (true) {
try {
var response = AdminReports.CustomerUsageReports.get(dateString,{parameters : "accounts:gsuite_basic_total_licenses,accounts:gsuite_basic_used_licenses"});
break;
} catch(e) {
//Logger.log(e.warnings.toString());
tryDate.setDate(tryDate.getDate()-1);
dateString = tryDate.getFullYear().toString() + "-" + (tryDate.getMonth() + 1).toString() + "-" + tryDate.getDate().toString();
continue;
}
};
var availLicenseCount = response.usageReports[0].parameters[0].intValue;
var usedLicenseCount = response.usageReports[0].parameters[1].intValue;
Logger.log("Available licenses:" + availLicenseCount.toString());
Logger.log("Used licenses:" + usedLicenseCount.toString());
return availLicenseCount;
}
I would recommend exploring GAM which is a tool that gives command line access to the administration functions of your domain.

Fiddler: Programmatically add word to Query string

Please be kind, I'm new to Fiddler
My purpose:I want to use Fiddler as a Google search filter
Summary:
I'm tired of manually adding "dog" every time I use Google.I do not want the "dog" appearing in my search results.
For example:
//www.google.com/search?q=cat+-dog
//www.google.com/search?q=baseball+-dog
CODE:
dog replaced with -torrent-watch-download
// ==UserScript==
// #name Tamper with Google Results
// #namespace http://superuser.com/users/145045/krowe
// #version 0.1
// #description This just modifies google results to exclude certain things.
// #match http://*.google.com
// #match https://*.google.com
// #copyright 2014+, KRowe
// ==/UserScript==
function GM_main () {
window.onload = function () {
var targ = window.location;
if(targ && targ.href && targ.href.match('https?:\/\/www.google.com/.+#q=.+') && targ.href.search("/+-torrent/+-watch/+-download")==-1) {
targ.href = targ.href +"+-torrent+-watch+-download";
}
};
}
//-- This is a standard-ish utility function:
function addJS_Node(text, s_URL, funcToRun, runOnLoad) {
var D=document, scriptNode = D.createElement('script');
if(runOnLoad) scriptNode.addEventListener("load", runOnLoad, false);
scriptNode.type = "text/javascript";
if(text) scriptNode.textContent = text;
if(s_URL) scriptNode.src = s_URL;
if(funcToRun) scriptNode.textContent = '(' + funcToRun.toString() + ')()';
var targ = D.getElementsByTagName('head')[0] || D.body || D.documentElement;
targ.appendChild(scriptNode);
}
addJS_Node (null, null, GM_main);
At first I was going to go with Tampermonkey userscripts,Because I did not know about Fiddler
==================================================================================
Now,lets focus on Fiddler
Before Request:
I want Fiddler to add text at the end of Google Query string.
Someone suggested me to use
static function OnBeforeRequest(oSession: Session) {
if (oSession.uriContains("targetString")) {
var sText = "Enter a string to append to a URL";
oSession.fullUrl = oSession.fullUrl + sText;
}
}
Before Response:
This is where my problem lies
I totally love the HTML response,Now I just want to scrape/hide the word in the search box without changing the search results.How can it be done? Any Ideas?
http://i.stack.imgur.com/4mUSt.jpg
Can you guys please take the above information and fix the problem for me
Thank you
Basing on goal definition above, I believe you can achieve better results with your own free Google custom search engine service. In particular, because you have control over GCSE fine-tuning results, returned by regular Google search.
Links:
https://www.google.com/cse/all
https://developers.google.com/custom-search/docs/structured_search

Eventbrite API date range parameter for organizer_list_events

I need a way to search via the eventbrite api past events, by organizer, that are private, but I also need to be able to limit the date range. I have not found a viable solution for this search. I assume the organizer_list_events api would be the preferred method, but the request paramaters don't seem to allow for the date range, and I am getting FAR too many returns.
I'm having some similar issues I posted a question to get a response about parsing the timezone, here's the code I'm using to get the dates though and exclude any events before today (unfortunately like you said I'm still getting everything sent to me and paring things out client side)
Note this is an AngularJS control but the code is just using the EventBrite javascript API.
function EventCtrl($http, $scope)
{
$scope.events=[];
$scope.noEventsDisplay = "Loading events...";
Eventbrite({'app_key': "EVC36F6EQZZ4M5DL6S"}, function(eb){
// define a few parameters to pass to the API
// Options are listed here: http://developer.eventbrite.com/doc/organizers/organizer_list_events/
//3877641809
var options = {
'id' : "3588304527",
};
// provide a callback to display the response data:
eb.organizer_list_events( options, function( response ){
validEvents = [];
var now = new Date().getTime();
for(var i = 0; i<response.events.length; i++)
{
var sd = response.events[i].event.start_date;
var ed = response.events[i].event.end_date;
var parsedSD = sd.split(/[:-\s]/);
var parsedED = ed.split(/[:-\s]/);
var startDate = new Date(parsedSD[0], parsedSD[1]-1, parsedSD[2], parsedSD[3], parsedSD[4], parsedSD[5]);
var endDate = new Date(parsedED[0], parsedED[1]-1, parsedED[2], parsedED[3], parsedED[4], parsedED[5]);
if(endDate.getTime()<now)
continue;
response.events[i].event.formattedDate = date.toDateString();
validEvents.push(response.events[i])
}
if(validEvents.length == 0)
{
$scope.$apply(function(scope){scope.noEventsDisplay = "No upcoming events to display, please check back soon.";});
}
else
{
$scope.$apply(function(scope){scope.noEventsDisplay = "";});
}
$scope.$apply(function(scope){scope.events = validEvents;});
//$('.event_list').html(eb.utils.eventList( response, eb.utils.eventListRow ));
});
});
}