// This script runs a TURF analysis and writes the results to a text item.
includeWeb('QScript Selection Functions');
var turf_question_name = 'TURF';
var heading_style = { size: 14 };
var table_style = { font: 'Lucida Console' };
if (!main())
log('QScript was cancelled.');
function main() {
var version = fileFormatVersion();
if (version < 8.20) {
alert('This script is not able to run on this version of Q.');
return false;
}
var data_file = requestOneDataFileFromProject(false);
if (data_file == null) {
alert('There is no data file in the project.');
return false;
}
var turf;
var turf_question;
var question_names;
var filters;
var weight;
var number_of_alternatives;
var alternative_labels;
var excel_friendly;
do {
// Input variables
var pick_any_questions = data_file.questions.filter(isQuestionPickAnyOrPickAnyGrid).filter(isQuestionNotHidden).filter(isQuestionValid).filter(isNotTurfQuestion).filter(function (q) { return !q.isBanner; });
if (pick_any_questions.length == 0) {
alert('There are no Pick Any questions in this project.');
return false;
}
var pick_any_variables = getVariablesFromQuestions(pick_any_questions);
var selected_variables;
do {
selected_variables = selectManyVariablesByQuestionNameAndLabel('Select the alternatives (variables) to be included in the TURF analysis.' +
' Only variables from Pick Any and Pick Any - Grid questions can be selected:', pick_any_variables).variables;
} while (!checkTwoOrMoreVariables(selected_variables))
// The alternative labels are "cleaned" by Q so they will be different from those in the input variables. We obtain the alternative labels from a TURF object.
number_of_alternatives = selected_variables.length;
turf_question = createQuestionWithLinkedVariables(turf_question_name, selected_variables, data_file, 'Pick Any');
var full_turf = createTurf(turf_question, null, null);
full_turf.maximumPortfolioSize = number_of_alternatives;
full_turf.minimumPortfolioSize = number_of_alternatives;
full_turf.optimizationMethod = 'Stochastic';
if(!checkTurfNotFailed(full_turf))
continue;
alternative_labels = full_turf.optimalPortfolios(number_of_alternatives)[0].alternativeLabels;
// Check that alternative labels are different from each other
if (!checkLabelsNotDuplicate(alternative_labels))
return false;
// Filter questions
var all_filter_variables = data_file.variables.filter(function (x) { return x.question.isFilter; });
if (all_filter_variables.length > 0 && askYesNo('Would you like to apply filters to the TURF analysis?'))
filters = selectManyVariablesByLabel('Select the filter variables to be included in the TURF analysis.', all_filter_variables).variables;
else
filters = null;
// Weight question
var all_weight_variables = data_file.variables.filter(function (x) { return x.question.isWeight; });
if (all_weight_variables.length > 0 && askYesNo('Would you like to apply a weight to the TURF analysis?'))
weight = selectOneVariableByLabel('Select the weight variable to be included in the TURF analysis.', all_weight_variables, false);
else
weight = null;
turf = createTurf(turf_question, filters, weight);
question_names = [];
selected_variables.forEach(function (x) {
if (question_names.indexOf(x.question.name) == -1)
question_names.push(x.question.name);
});
} while (!turf || !checkTurfNotFailed(turf))
// Create basic TURF object to retrieve reach and frequency of each alternative
var basic_turf = createTurf(turf_question, filters, weight);
basic_turf.maximumPortfolioSize = 1;
basic_turf.numberOfPortfolios = number_of_alternatives;
var size_one_portfolios = basic_turf.optimalPortfolios(1);
// Advanced options
var excluded_labels = [];
var forced_labels = [];
var max_max_portfolio_size = number_of_alternatives;
if (askYesNo('Would you like to modify advanced options?', 'Total_Unduplicated_Reach_and_Frequency_(TURF)')) {
var min_min_portfolio_size = 1;
// Forced alternatives
if (askYesNo('Would you like to specify forced alternatives (i.e. alternatives that must be included in all portfolios)?', 'Total_Unduplicated_Reach_and_Frequency_(TURF)#Forced_Alternatives')) {
turf.forcedAlternatives = selectManyVariablesByLabel('Select the forced alternatives.', turf_question.variables).variables;
if (turf.forcedAlternatives.length > 1) {
min_min_portfolio_size = turf.forcedAlternatives.length;
alert('The minimum possible portfolio size has increased to ' + min_min_portfolio_size + ' due to the inclusion of forced alternatives.');
}
if (turf.forcedAlternatives.length > 0 && fileFormatVersion() == 8.23) // there is a bug with forced alternatives using the exhaustive method
turf.optimizationMethod = 'Stochastic';
}
// Mutually exclusive alternatives
if (askYesNo('Would you like to specify mutually exclusive alternatives (i.e. alternatives that must not appear with each other).',
'Total_Unduplicated_Reach_and_Frequency_(TURF)#Mutually_Exclusive_Alternatives')) {
var mutually_exclusive_alternatives = [];
do {
var selected_alternatives = selectManyVariablesByLabel('Select alternatives that must be mutually exclusive:', turf_question.variables).variables;
if (checkTwoOrMoreVariables(selected_alternatives) &&
checkMutuallyExclusiveAlternativesNotForced(turf.forcedAlternatives, selected_alternatives)) {
mutually_exclusive_alternatives.push(selected_alternatives);
if (number_of_alternatives == selected_alternatives.length) {
alert('All alternatives have been selected to be mutually exclusive. There will only be portfolios of size one.');
break;
}
if (!askYesNo('Would you like to specify an additional set of mutually exclusive variables?', 'Total_Unduplicated_Reach_and_Frequency_(TURF)#Mutually_Exclusive_Alternatives'))
break;
}
} while (true)
turf.mutuallyExclusiveAlternatives = mutually_exclusive_alternatives;
if (fileFormatVersion() <= 8.20 && mutually_exclusive_alternatives.length > 1) // difficult to calculate the exact max max portfolio size for this case, which is necessary for the stochastic method.
turf.optimizationMethod = 'Exhaustive';
}
// Minimum proportion of positive responses
while (true) {
var min_proportion = prompt('Enter the minimum proportion of positive responses (from 0 to 1). Alternatives whose proportion of positive responses is less than this number will be excluded.', 0,
'Total_Unduplicated_Reach_and_Frequency_(TURF)#Minimum_Proportion_of_Positive_Responses');
if (min_proportion < 0)
alert('Enter a proportion greater than or equal to zero.');
else if (min_proportion > 1)
alert('Enter a proportion less than or equal to one.');
else {
var excluded = size_one_portfolios.filter(function (x) { return x.reach < min_proportion; });
excluded_labels = excluded.map(function (x) { return x.alternativeLabels[0]; });
forced_labels = turf.forcedAlternatives.map(function (x) { return x.label.trim(); });
var forced_and_excluded = excluded.filter(function (x) { return forced_labels.indexOf(x.alternativeLabels[0]) != -1; });
if (number_of_alternatives == excluded.length)
alert('The input value is too high as all alternatives would be excluded. Enter a smaller proportion.');
else if (forced_and_excluded.length > 0) {
var message = 'The input value is too high as the following forced variables would be excluded. Enter a smaller proportion.\n\n';
forced_and_excluded.forEach(function (x) { message += x.alternativeLabels[0] + '\n'; });
alert(message);
} else {
turf.minimumProportionOfPositiveResponses = min_proportion;
if (excluded_labels.length == 0 && min_proportion > 0)
alert('No alternatives have been excluded as their minimum proportions are all at least ' + min_proportion + '.');
else if (excluded_labels.length == 1)
alert('The following alternative has been excluded as its minimum proportion is below ' + min_proportion + ':\n\n' + excluded_labels[0]);
else if (excluded_labels.length > 1)
alert('The following alternatives have been excluded as their minimum proportion is below ' + min_proportion + ':\n\n' + excluded_labels.join('\n'));
break;
}
}
}
// Set max max portfolio size given the mutually exclusive alternatives and excluded alternatives
max_max_portfolio_size = number_of_alternatives - excluded_labels.length;
turf.mutuallyExclusiveAlternatives.forEach(function (x) {
var mutually_exclusive_labels = x.map(function (y) { return y.label.trim(); });
var remaining = alternative_labels.filter(function (label) {
return excluded_labels.indexOf(label) == -1 && mutually_exclusive_labels.indexOf(label) == -1;
});
max_max_portfolio_size = Math.min(max_max_portfolio_size, remaining.length + 1);
});
// Minimum alternatives per case
if (max_max_portfolio_size > 1) {
var min_alternatives_per_case;
while (true) {
min_alternatives_per_case = prompt('Enter the minimum alternatives per case (1 to ' + max_max_portfolio_size +
'). This is the minimum number of selected alternatives in each case' +
' in order for the case to be counted in the portfolio\'s reach.', 1,
'Total_Unduplicated_Reach_and_Frequency_(TURF)#Reach_and_the_Minimum_Alternatives_Per_Case');
if (isNaN(min_alternatives_per_case) || min_alternatives_per_case != Math.round(min_alternatives_per_case))
alert('Enter a whole number (e.g. 1).');
else if (min_alternatives_per_case <= 0)
alert('Enter a number greater than zero.');
else if (min_alternatives_per_case > max_max_portfolio_size)
alert('The input value is too high as there will be no portfolios with a positive reach. Enter a smaller number.');
else
break;
}
turf.minimumAlternativesPerCase = Math.round(min_alternatives_per_case);
if (turf.minimumAlternativesPerCase > min_min_portfolio_size) {
min_min_portfolio_size = turf.minimumAlternativesPerCase;
alert('The minimum possible portfolio size has increased to ' + min_min_portfolio_size + ' as this is the minimum number of alternatives per case.');
}
}
// Minimum portfolio size
var min_portfolio_size;
if (min_min_portfolio_size == max_max_portfolio_size) {
turf.maximumPortfolioSize = max_max_portfolio_size;
turf.minimumPortfolioSize = max_max_portfolio_size;
} else {
while (true) {
min_portfolio_size = prompt('Enter the minimum portfolio size (' + min_min_portfolio_size + ' to ' + max_max_portfolio_size +
'). Only portfolios of this size and larger will be displayed.', min_min_portfolio_size,
'Total_Unduplicated_Reach_and_Frequency_(TURF)#Minimum_Portfolio_Size');
if (isNaN(min_portfolio_size) || min_portfolio_size != Math.round(min_portfolio_size))
alert('Enter a whole number (e.g. 1).');
else if (min_portfolio_size <= 0)
alert('Enter a number greater than zero.');
else if (min_portfolio_size < turf.forcedAlternatives.length)
alert('Enter a number greater than or equal to the number of forced alternatives (' + turf.forcedAlternatives.length + ').');
else if (min_portfolio_size < min_min_portfolio_size)
alert('Enter a number greater than or equal to ' + min_min_portfolio_size + '. Otherwise there will be some portfolio sizes with no valid portfolios.');
else if (min_portfolio_size > max_max_portfolio_size)
alert('Enter a number less than or equal to ' + max_max_portfolio_size + '. Otherwise there will be no valid portfolios.');
else
break;
}
turf.maximumPortfolioSize = max_max_portfolio_size; // need to set maximumPortfolioSize at least as high as minimumPortfolioSize to avoid exception
turf.minimumPortfolioSize = Math.round(min_portfolio_size);
}
}
// Maximum portfolio size
if (turf.minimumPortfolioSize != max_max_portfolio_size) {
var max_portfolio_size;
while (true) {
max_portfolio_size = prompt('Enter the maximum portfolio size (' + turf.minimumPortfolioSize + ' to ' + max_max_portfolio_size +
'). Portfolios of size ' + turf.minimumPortfolioSize + ' to this number will be displayed.',
Math.min(max_max_portfolio_size, Math.max(turf.minimumPortfolioSize, 8)),
'Total_Unduplicated_Reach_and_Frequency_(TURF)#Maximum_Portfolio_Size');
if (isNaN(max_portfolio_size) || max_portfolio_size != Math.round(max_portfolio_size))
alert('Enter a whole number (e.g 1).');
else if (max_portfolio_size < turf.minimumPortfolioSize)
alert('Enter a number greater than or equal to the minimum portfolio size (' + turf.minimumPortfolioSize + ').');
else if (max_portfolio_size > max_max_portfolio_size)
alert('Enter a number less than or equal to ' + max_max_portfolio_size + '. Otherwise there will be some portfolio sizes with no valid portfolios.');
else
break;
}
turf.maximumPortfolioSize = Math.round(max_portfolio_size);
}
// Number of top portfolios to display
var number_of_portfolios;
while (true) {
number_of_portfolios = prompt('Enter the number of top portfolios to display for each portfolio size.' +
' If left at 1, only the best portfolio will be shown.' +
' If set to 2, then the best and second best portfolio will be shown, etc.', 1,
'Total_Unduplicated_Reach_and_Frequency_(TURF)#Number_of_Top_Portfolios_to_Display');
if (isNaN(number_of_portfolios) || number_of_portfolios != Math.round(number_of_portfolios))
alert('Enter a whole number (e.g. 1).');
else if (number_of_portfolios <= 0)
alert('Enter a number greater than zero.');
else
break;
}
turf.numberOfPortfolios = Math.round(number_of_portfolios);
// Optimization method
if (turf.optimizationMethod == 'Default' && turf.numberOfCombinations >= turf.thresholdNumberOfCombinations) {
var message = 'This TURF analysis may take more than a few seconds to run with the usual "Exhaustive" method,' +
' would you like to use the "Stochastic" method instead, which takes less time and is still' +
' highly likely to return the same results?';
if (!askYesNo(message, 'Total_Unduplicated_Reach_and_Frequency_(TURF)#Optimization_Methods'))
turf.optimizationMethod = 'Exhaustive';
}
if (turf.optimizationMethod == 'Exhaustive' && turf.numberOfCombinations * turf.baseN > Math.pow(10, 9))
alert('The requested TURF analysis may take more than a few seconds to run. Are you sure you want to proceed?');
else if (turf.numberOfCombinations * turf.baseN > Math.pow(10, 12))
alert('The requested TURF analysis may take more than a few seconds to run. Are you sure you want to proceed?');
excel_friendly = askYesNo('Format text for cutting and pasting to Excel?');
var text_item = project.report.appendText();
if (project.report.setSelectedRaw) // Remove when Q 4.10 is stable
project.report.setSelectedRaw([text_item]);
// Title
var title_builder = Q.htmlBuilder();
title_builder.appendParagraph(turf_question.name + ' (' + number_of_alternatives + ' alternatives from ' + question_names.join(' and ') + ')', { font: 'Tahoma', size: 20});
text_item.title = title_builder;
// Body
var builder = Q.htmlBuilder();
builder.setStyle({ font: 'Tahoma', size: 10 });
// Summary table
builder.appendParagraph('TURF Summary: Best Portfolios for Each Size', heading_style);
var summary_table = [[excel_friendly ? 'Portfolio' : '', 'Size', 'Reach', 'Freq.']];
for (var i = turf.minimumPortfolioSize; i <= turf.maximumPortfolioSize; i++) {
var portfolios = turf.optimalPortfolios(i);
if (portfolios.length > 0)
summary_table.push([portfolios[0].alternativeLabels.join(' OR '), i, Q.DecimalsToShow(100 * portfolios[0].reach, 2) + '%', Q.DecimalsToShow(portfolios[0].frequency, 0)]);
else
summary_table.push(['No portfolios for this size', i, '', '']);
}
summary_table.push(['', '', '', '']);
if (excel_friendly) {
summary_table = moveColumnToLast(summary_table, 0);
appendTabTable(builder, summary_table, table_style);
} else
builder.appendTable(summary_table, [75, 5, 8, 12], '-', table_style);
// Tables for each portfolio size
if (turf.numberOfPortfolios > 1) {
for (var i = turf.minimumPortfolioSize; i <= turf.maximumPortfolioSize; i++) {
builder.appendParagraph('Best Portfolios of Size ' + i, heading_style);
var portfolios = turf.optimalPortfolios(i);
var table = [[excel_friendly ? 'Portfolio' : '', 'Rank', 'Reach', 'Freq.']];
if (portfolios.length == 0)
table.push(['No portfolios for this size', '', '', '']);
else {
for (var j = 0; j < portfolios.length; j++)
table.push([portfolios[j].alternativeLabels.join(' OR '), j + 1, Q.DecimalsToShow(100 * portfolios[j].reach, 2) + '%', Q.DecimalsToShow(portfolios[j].frequency, 0)]);
}
table.push(['', '', '', '']);
if (excel_friendly) {
table = moveColumnToLast(table, 0);
appendTabTable(builder, table, table_style);
} else
builder.appendTable(table, [75, 5, 8, 12], '-', table_style);
}
}
// Alternatives
builder.appendParagraph('List of All ' + number_of_alternatives + ' Alternatives from ' + question_names.join(' and '), heading_style);
var alternatives_table = [['', '%', 'Population']];
var size_one_labels = size_one_portfolios.map(function (y) { return y.alternativeLabels[0]; });
alternative_labels.forEach(function (label) {
var portfolio_index = size_one_labels.indexOf(label);
alternatives_table.push([label,
Q.DecimalsToShow(100 * size_one_portfolios[portfolio_index].reach, 2) + '%',
Q.DecimalsToShow(size_one_portfolios[portfolio_index].frequency, 0)]);
});
alternatives_table.push(['', '', '']);
if (excel_friendly) {
alternatives_table = moveColumnToLast(alternatives_table, 0);
appendTabTable(builder, alternatives_table, table_style);
} else
builder.appendTable(alternatives_table, [82, 8, 12], '-', table_style);
// Forced alternatives
if (turf.forcedAlternatives.length > 0) {
builder.appendParagraph('Forced Alternatives', heading_style);
var forced_table = [['']];
turf.forcedAlternatives.forEach(function (x) {
forced_table.push([getVariableLabel(x)]);
});
forced_table.push(['']);
if (excel_friendly)
appendTabTable(builder, forced_table, table_style);
else
builder.appendTable(forced_table, [82], '-', table_style);
}
// Mutually exclusive alternatives
turf.mutuallyExclusiveAlternatives.forEach(function (x, index) {
if (turf.mutuallyExclusiveAlternatives.length == 1)
builder.appendParagraph('Mutually Exclusive Alternatives', heading_style);
else
builder.appendParagraph('Mutually Exclusive Alternatives ' + (index + 1), heading_style);
var mutually_exclusive_table = [['']];
x.forEach(function (y) {
mutually_exclusive_table.push([getVariableLabel(y)]);
});
mutually_exclusive_table.push(['']);
if (excel_friendly)
appendTabTable(builder, mutually_exclusive_table, table_style);
else
builder.appendTable(mutually_exclusive_table, [82], '-', table_style);
});
// Note about minimum proportion
if (excluded_labels.length > 0) {
builder.appendParagraph('Excluded Alternatives', heading_style);
var excluded_table = [['']];
excluded_labels.forEach(function (label) { excluded_table.push([label]); });
excluded_table.push(['']);
builder.appendParagraph('Alternatives with % less than ' + (100 * turf.minimumProportionOfPositiveResponses) + '%');
if (excel_friendly)
appendTabTable(builder, excluded_table, table_style);
else
builder.appendTable(excluded_table, [82], '-', table_style);
}
builder.appendParagraph('Note', heading_style);
builder.appendParagraph('Portfolios are ranked in terms of highest reach and then highest frequency.');
// Note about minimum alternatives per case
if (turf.minimumAlternativesPerCase > 1)
builder.appendParagraph('The minimum number of alternatives per case is ' + turf.minimumAlternativesPerCase + '.');
// Note about filters
if (filters != null && filters.length > 0) {
if (filters.length == 1)
builder.appendParagraph('The filter "' + filters[0].label + '" has been applied to the TURF analysis.');
else {
builder.appendParagraph('The following filters have been applied to the TURF analysis:');
var bulleted_list = [];
for (var i = 0; i < filters.length; i++)
bulleted_list.push(filters[i].label);
builder.appendBulletted(bulleted_list);
}
}
// Note about weight
if (weight != null)
builder.appendParagraph('The weight "' + weight.label + '" has been applied to the TURF analysis.');
// Notes about sample size
builder.appendParagraph('Base n (unweighted total sample size): ' + Q.DecimalsToShow(turf.baseN, 0));
if (weight != null)
builder.appendParagraph('Base Population (weighted total sample size): ' + Q.DecimalsToShow(turf.basePopulation, 0));
builder.appendParagraph('Missing n (number of observations excluded due to missing data): ' + Q.DecimalsToShow(turf.missingN, 0), turf.missingN > 0 ? { color: 'red' } : null);
// Note about optimization method
if (turf.optimizationMethod == 'Default' && turf.numberOfCombinations >= turf.thresholdNumberOfCombinations)
builder.appendParagraph('A stochastic optimization algorithm was used to compute the optimal portfolios due to the large number of combinations that were evaluated in this TURF analysis.');
// Creation timestamp
builder.appendParagraph('Created: ' + new Date().toLocaleString());
text_item.content = builder;
log('A text item called ' + text_item.name + ' has been added to the report.');
return true;
// Obtain alternative label given linked variable
function getVariableLabel(variable) {
for (var i = 0; i < number_of_alternatives; i++)
if (turf_question.variables[i].name == variable.name)
return alternative_labels[i];
}
}
function isQuestionPickAnyOrPickAnyGrid(question) {
var q_type = question.questionType;
return q_type == 'Pick Any' || q_type == 'Pick Any - Grid';
}
function isNotTurfQuestion(question) {
return !question.name.match('^' + turf_question_name + '(| \\d+)$');
}
function checkTwoOrMoreVariables(variables) {
if (variables.length >= 2)
return true;
alert('There needs to be two or more variables.\n\nClick OK to reselect variables or Cancel to stop the script.');
return false;
}
function checkTurfNotFailed(turf) {
if (turf.failed) {
alert('The inputs are not valid:\n\n' + turf.failureMessage + '\n\nClick OK to reselect alternatives or Cancel to stop the script.');
return false;
} else
return true;
}
function checkLabelsNotDuplicate(labels) {
if (labels.length <= 1)
return true;
for (var i = 1; i < labels.length; i++) {
for (var j = 0; j < i; j++) {
if (labels[i] == labels[j]) {
alert('The selected alternatives have labels that are identical or too similar to each other. Change the labels or select different alternatives.');
return false;
}
}
}
return true;
}
function checkMutuallyExclusiveAlternativesNotForced(forced_alternatives, mutually_exclusive_alternatives) {
var both_forced_and_mutually_exclusive = [];
var mutually_exclusive_alternatives_names = mutually_exclusive_alternatives.map(function (x) { return x.name; });
forced_alternatives.forEach(function (x) {
if (mutually_exclusive_alternatives_names.indexOf(x.name) != -1)
both_forced_and_mutually_exclusive.push(x.label);
});
if (both_forced_and_mutually_exclusive.length < 2)
return true;
else {
alert('The selected mutually exclusive alternatives are not valid as they contain the following forced alternatives:\n\n' +
both_forced_and_mutually_exclusive.join('\n'));
return false;
}
}
// Tab separated columns that can be copied into Excel
function appendTabTable(builder, cells, style) {
cells.forEach(function (row) {
builder.appendParagraph(row.join(String.fromCharCode(9)), style);
});
}
// The first column often has long names that should be moved to the back
// to improve readability for tab tables
function moveColumnToLast(table, col) {
var new_table = [];
table.forEach(function (row) {
var new_row = row;
new_row.push(row[col]);
new_row.splice(col, 1);
new_table.push(new_row);
});
return new_table;
}