QScript Functions for Combining Categories
Jump to navigation
Jump to search
This page that contains functions that are helpful for combining categries, either by creating NETs on tables or by creating new variables.
To make these functions available when writing a QScript or Rule see JavaScript Reference.
createTopBoxQuestion(question, value_array)
Create a new Pick Any question from the input question. The new question will have the values specified in the value_array selected in the Count This Value column of the Values. The new question will take the same name as the input question with the labels of the categories that have been counted appended to the end of the name.
createBottomBoxQuestion(question, value_array)
Create a new Pick Any question from the input question. The new question will have the values specified in the value_array selected in the Count This Value column of the Values. The new question will take the same name as the input question with the labels of the categories that have been counted appended to the end of the name.
createKBoxNETs(selected_questions, k, bottom, position, name_override)
This function adds top or bottom box NET categories to the data reductions of the questions in the array selected_questions.
The arguments are:
- selected_questions is the array of questions to change.
- k is the number of categories to combine. This should be a positive integer.
- bottom is a boolean flag. If true, then bottom box NETs will be created, else top box NETs will be created.
- position is a string with the possible values: Beginning, End, Default position. This determines whether the new NET is move to the beginning (top or left) or end (bottom or right) of the data reduction, or if it is left in its initial position.
pickOneMultiToPickAnyFlattenAndMergeByRows(question, mergings, use_source)
Create a new Pick Any question from a Pick One - Multi question such that scale points are merged according to mergings, and the rows of the Pick One - Multi are flattened in the usual sense.
The mergings argument is an array whose elements are objects with properties:
- name - a string corresponding to the name of the merged category.
- values - an array containing the source values that are contained in the merged category.
use_source is a boolean flag. When set to true then the new variables will refer to source values of the variables in the input question, and otherwise they will refer to the present values in the input question.
mergePickOneExpression(var_name, values_to_merge, use_source)
Generates a JavaScript expression for merging the values_to_merge from the variable with name var_name. If use_source then source values are used, otherwise the present values are used.
topAndBottomBoxNETCreator(bottom)
This function presents a common user interface for adding Top/Bottom Box NETs to questions. Set bottom to true for Bottom Boxes, and false for Top Boxes. The user can specify the number of categories to combine, the position of the new NET, and the label of the new NET. The determination of highest (or lowest) is done based on the current values of the categories in the data reduction (so recoding affects this).
getTopOrBottomKRecodedCategories(question, k, bottom)
Find the k highest or lowest categories in the question according to the current values. Set bottom to true for the lowest categories, and false for the highest.
createTopOrBottomBoxVariables(k, is_bottom)
Function for creating top and bottom box variables. Provides the user prompts and creates variables.
- k is the number of categories to combine. If you want the user to enter this number, set k as null.
- is_bottom: set to true for bottom boxes and false for top boxes
mergeScales(mergings, merge_message)
Merge the points in a scale.
- mergings is an array where each element has properties label, a string which determines the new label of the merged category, and values, an array of integers specifying the source values of the categories to be merged.
- merge_message is a string describing the mergings that will be done.
mergeCategoriesInManyQuestions(selected_questions, nets)
tabulateMergedQuestions(new_net_questions, invalid_questions, merge_message)
applyCustomMerges(create_new_pick_any_question)
Source Code
includeWeb('QScript Utility Functions');
includeWeb('QScript Data Reduction Functions');
includeWeb('QScript Value Attributes Functions');
// TOP/BOTTOM BOXES
// Create a Top Box question from the input question, using the values in value_array as
// the top values to be selected.
function createTopBoxQuestion(question, value_array) {
checkQuestionType(question, ["Pick One", "Pick One - Multi"]);
var unique_values = question.uniqueValues;
var n_uniques = unique_values.length;
var combined_labels = getLabelsForValues(question, value_array);
var missing_values_indicators = valueAttributesMissingValueIndicatorArray(question);
var name = question.name;
var new_name = name + " - " + combined_labels.join(" + ");
var new_question = question.duplicate(preventDuplicateQuestionName(question.dataFile, new_name));
new_question.questionType = "Pick Any";
var v = new_question.variables;
if (v.length === 1)
v[0].label = new_question.name;
for (var j = 0; j < n_uniques; j++) {
setIsMissingDataForVariablesInQuestion(new_question, unique_values[j], missing_values_indicators[j] == 1)
setCountThisValueForVariablesInQuestion(new_question, unique_values[j], value_array.indexOf(unique_values[j]) > -1)
}
new_question.needsCheckValuesToCount = false;
return new_question;
}
// Create a Bottom Box question from the input question, using the values in value_array as
// the Bottom values to be selected.
function createBottomBoxQuestion(question, value_array) {
checkQuestionType(question, ["Pick One", "Pick One - Multi"]);
var unique_values = question.uniqueValues;
var n_uniques = unique_values.length;
var combined_labels = getLabelsForValues(question, value_array);
var missing_values_indicators = valueAttributesMissingValueIndicatorArray(question);
var name = question.name;
var new_name = name + " - " + combined_labels.join(" + ");
var new_question = question.duplicate(preventDuplicateQuestionName(question.dataFile, new_name));
new_question.questionType = "Pick Any";
var v = new_question.variables;
if (v.length === 1)
v[0].label = new_question.name;
for (var j = 0; j < n_uniques; j++) {
setIsMissingDataForVariablesInQuestion(new_question, unique_values[j], missing_values_indicators[j] == 1)
setCountThisValueForVariablesInQuestion(new_question, unique_values[j], value_array.indexOf(unique_values[j]) > -1)
}
new_question.needsCheckValuesToCount = false;
return new_question;
}
// Creates a top or bottom box NET in each of the questions in the array selected_questions
function createKBoxNETs(selected_questions, k, bottom, position, name_override) {
var num_selected_questions = selected_questions.length;
// Add NETs to the data reductions as specified
var current_question;
var current_data_reduction;
var n_values;
var values_for_net;
var net_labels;
var new_net_questions = [];
var check_questions = [];
var invalid_questions = [];
var destination_label;
var net_name;
// Loop through selected questions, creating NETs where possible
for (var j = 0; j < num_selected_questions; j++) {
current_question = selected_questions[j];
current_data_reduction = current_question.dataReduction;
n_values = current_question.uniqueValues.length - numberMissingValues(current_question);
if (bottom)
values_for_net = getBottomKNonMissingValues(current_question, k);
else
values_for_net = getTopKNonMissingValues(current_question, k);
net_labels = getLabelsForValues(current_question, values_for_net);
// check that the top/bottom category labels still exist in the data reduction
if (dataReductionContainsLabels(current_question, net_labels)) {
// Determine the name for the NET
if (name_override != null)
net_name = name_override;
else if (bottom)
net_name = "Bottom " + k + " NET (" + net_labels.join(', ') + ")";
else
net_name = "Top " + k + " NET (" + net_labels.join(', ') + ")";
// Create the NET
try {
current_data_reduction.createNET(net_labels, net_name);
}
catch (e) {
log ("Could not create " + net_name + " from '" +
question.name + "': " + e);
return false;
}
// Determine where to place the new NET row (or column)
if (position == "Beginning")
destination_label = null;
else {
var row_labels = current_data_reduction.rowLabels;
var col_labels = current_data_reduction.columnLabels;
var net_rows = current_data_reduction.netRows;
var net_columns = current_data_reduction.netColumns;
if (net_rows) {
if (net_rows.length > 0) {
if (net_rows[net_rows.length - 1] == row_labels.length - 1)
destination_label = row_labels[row_labels.length - 2];
else
destination_label = row_labels[row_labels.length - 1];
} else if (col_labels != null && net_columns.length > 0) {
if (net_columns[net_columns.length - 1] == col_labels.length - 1)
destination_label = col_labels[col_labels.length - 2];
else
destination_label = col_labels[col_labels.length - 1];
}
} else {
if (row_labels.indexOf(net_name) > -1) {
if (row_labels.indexOf('NET') == row_labels.length - 1)
destination_label = row_labels[row_labels.length - 2];
else
destination_label = row_labels[row_labels.length - 1];
} else if (col_labels != null && col_labels.indexOf(net_name) > -1) {
if (col_labels.indexOf('NET') == col_labels.length - 1)
destination_label = col_labels[col_labels.length - 2];
else
destination_label = col_labels[col_labels.length - 1];
}
}
}
// Move the NET
if (position != "Default position")
current_data_reduction.moveAfter(net_name, destination_label);
new_net_questions.push(current_question);
} else
invalid_questions.push(current_question);
if (questions_containing_dk.indexOf(current_question) > -1)
check_questions.push(current_question);
}
return {netQuestions: new_net_questions, invalidQuestions: invalid_questions, checkQuestions: check_questions};
}
// Create a new Pick Any question from a Pick One - Multi question such that scale points are
// merged according to 'mergings', and the rows of the Pick One - Multi are flattened in the
// usual sense.
function pickOneMultiToPickAnyFlattenAndMergeByRows(question, mergings, use_source, simplify_code_labels) {
if (simplify_code_labels == null)
simplify_code_labels = true;
checkQuestionType(question, ["Pick One - Multi"]);
var new_vars = [];
var q_vars = question.variables;
var last_var = q_vars[q_vars.length - 1];
var n_mergings = mergings.length;
// Generate new JavaScript variables for the merged categories
q_vars.forEach(function (q_var) {
var cur_var_label = q_var.label;
var cur_var_name = q_var.name;
var name_prefix = cur_var_name + '_flat_';
for (var k = 0; k < n_mergings; k++) {
var new_expression = mergePickOneExpression(cur_var_name, mergings[k].values, use_source);
var new_label = cur_var_label + " - " + mergings[k].name;
var new_var = question.dataFile.newJavaScriptVariable(new_expression, false, preventDuplicateVariableName(question.dataFile, name_prefix + (k + 1)), new_label, last_var);
new_var.variableType = "Categorical";
last_var = new_var;
new_vars.push(new_var);
}
});
// setting the question
var new_q_name = preventDuplicateQuestionName(question.dataFile, question.name + " (flattened)");
var new_q = question.dataFile.setQuestion(new_q_name, "Pick Any", new_vars);
var v = new_q.variables;
if (v.length === 1)
v[0].label = new_q.name;
// creating the spans and simplifying the labels
for (var j = 0; j < q_vars.length; j++){
var labels_to_merge = [];
for (var k = 0; k < n_mergings; k++)
labels_to_merge.push(new_vars[k + j * n_mergings].label);
new_q.dataReduction.span(labels_to_merge, q_vars[j].label);
if (simplify_code_labels)
{
for (var k = 0; k < n_mergings; k++)
new_q.dataReduction.rename(labels_to_merge[k], mergings[k].name);
}
}
setCountThisValueForVariablesInQuestion(new_q, 1, true);
new_q.needsCheckValuesToCount = false;
return new_q;
}
// Generate the expression for merging a set of values
function mergePickOneExpression(var_name, values_to_merge, use_source) {
var expression ="Q.IsMissing(" + var_name + ") ? NaN : ";
for (var j = 0; j < values_to_merge.length; j++) {
cur_val = values_to_merge[j];
if (j > 0)
expression += " || ";
if (use_source) {
if (isNaN(cur_val))
expression += "isNaN(Q.Source(" + var_name + "))";
else
expression += "Q.Source(" + var_name + ") == " + cur_val;
} else {
if (isNaN(cur_val))
expression += expression += "isNaN(" + var_name + ")";
else
expression += var_name + " == " + cur_val;
}
}
return expression + " ? 1 : 0;";
}
function topAndBottomBoxNETCreator(bottom) {
includeWeb('QScript Utility Functions');
includeWeb('QScript Questionnaire Functions');
includeWeb('QScript Selection Functions');
includeWeb('QScript Value Attributes Functions');
includeWeb('QScript Data Reduction Functions');
includeWeb('QScript Functions for Processing Arrays');
includeWeb('JavaScript Utilities');
includeWeb('QScript Functions to Generate Outputs');
// Creates a top or bottom box NET in each of the questions in the array selected_questions
function createKBoxNETsForCreator(question_objects, k, bottom, position, name_override, cant_modify, remove_dk) {
var new_net_questions = [];
// Loop through selected questions and create NETs
question_objects.forEach(function (obj) {
var question = obj.question;
var current_data_reduction = question.dataReduction;
var net_labels = getTopOrBottomKRecodedCategories(question, k, bottom, { excludeDK: true });
// Check that it is possible to create the NET. If labels are not present
// then it is not possible to create the NET.
var keep_going = true;
if (net_labels.length == 0) {
cant_modify.push(question);
keep_going = false;
} else {
net_labels.forEach(function (label) {
if (!current_data_reduction.contains(label)) {
keep_going = false;
cant_modify.push(question);
}
});
}
if (keep_going) {
// Determine the name for the NET
var net_name;
if (name_override != null)
net_name = name_override;
else if (bottom)
net_name = "Bottom " + k + " NET (" + net_labels.join(', ') + ")";
else
net_name = "Top " + k + " NET (" + net_labels.join(', ') + ")";
// Create the NET
try {
current_data_reduction.createNET(net_labels, net_name);
}
catch (e) {
log ("Could not create " + net_name + " from '" +
question.name + "': " + e);
return false;
}
// Move the NET
var destination_label;
if (position != "Default position") {
// Determine where to place the new NET row (or column)
if (position == "Beginning")
destination_label = null;
else {
var row_labels = current_data_reduction.rowLabels;
var col_labels = current_data_reduction.columnLabels;
var net_rows = current_data_reduction.netRows;
var net_columns = current_data_reduction.netColumns;
if (net_rows) {
if (net_rows.length > 0) {
if (net_rows[net_rows.length - 1] == row_labels.length - 1)
destination_label = row_labels[row_labels.length - 2];
else
destination_label = row_labels[row_labels.length - 1];
} else if (col_labels != null && net_columns.length > 0) {
if (net_columns[net_columns.length - 1] == col_labels.length - 1)
destination_label = col_labels[col_labels.length - 2];
else
destination_label = col_labels[col_labels.length - 1];
}
} else {
if (row_labels.indexOf(net_name) > -1) {
if (row_labels.indexOf('NET') == row_labels.length - 1)
destination_label = row_labels[row_labels.length - 2];
else
destination_label = row_labels[row_labels.length - 1];
} else if (col_labels != null && col_labels.indexOf(net_name) > -1) {
if (col_labels.indexOf('NET') == col_labels.length - 1)
destination_label = col_labels[col_labels.length - 2];
else
destination_label = col_labels[col_labels.length - 1];
}
}
}
current_data_reduction.moveAfter(net_name, destination_label);
}
new_net_questions.push(question);
if (remove_dk && obj.hasDK) {
var value_attributes = question.valueAttributes;
var unique_values = question.uniqueValues;
unique_values.forEach(function (x) {
if (isDontKnow(value_attributes.getLabel(x)))
value_attributes.setIsMissingData(x, true);
});
}
}
});
return new_net_questions;
}
const is_displayr = inDisplayr();
let structure_name = is_displayr ? "variable sets" : "questions";
let allowed_types = ["Nominal", "Ordinal", "Nominal - Multi", "Ordinal - Multi", "Binary - Multi"];
let selected_questions;
if (is_displayr) {
let user_selections = getAllUserSelections();
selected_questions = user_selections.selected_questions.concat(user_selections.questions_in_rows_of_selected_tables);
selected_questions = selected_questions.filter(function (q) { return allowed_types.indexOf(q.variableSetStructure) > -1});
if (selected_questions.length == 0) {
log("No appropriate data selected. Select one or more "
+ printTypesString(allowed_types) + " " + structure_name
+ " and run this option again");
return false;
}
} else
selected_questions = selectInputQuestions(allowed_types, false, true, true);
if (!selected_questions) {
allowed_types = !is_displayr ? onlyUnique(convertStructureToType(allowed_types)) : allowed_types;
log("No appropriate " + structure_name + " have been selected. Please select appropriate " + printTypesString(allowed_types) + " " + structure_name + ".")
return false;
}
// Specify k
let k = -1;
while (!isPositiveInteger(k) || k < 2) {
k = prompt("How many categories do you wish to combine? For example, to combine the " + (bottom ? "bottom" : "top") + " two categories type '2' and press OK.");
if (!isPositiveInteger(k) || k < 2)
alert("k must be an integer greater than 1.");
}
let relevant_questions = selected_questions.filter(function(q) { return (q.uniqueValues.length - numberOfMissingOrNaNValues(q)) > k; } );
if (relevant_questions.length == 0) {
log("The selected " + structure_name + " contain less than " +
k + " or more categories with non missing data. Please select a smaller value of k or choose appropriate " + structure_name + " before re-running this script.");
return false;
}
let override_name = prompt("Enter a label for the" + (bottom ? " bottom " : " top ") + k + " NET. To keep the default label leave this blank.");
if (!/\S/.test(override_name))
override_name = null;
// Check relevant questions for 'Dont Know' options
let question_objects = relevant_questions.map(function (q) {
return { question: q,
hasDK: nonMissingValueLabels(q).filter(isDontKnow).length > 0 }
});
let dk_message = "Some of the selected questions contain 'Don't Know' categories. These categories will not be used to create the "
+ (bottom ? "bottom" : "top") + " " + k + " boxes. Do you also want to remove the 'Don't Know' categories?";
let remove_dk = false;
if (question_objects.some(function (obj) { return obj.hasDK; }))
remove_dk = askYesNo(dk_message);
let position_options = ["Beginning", "End", "Default position"];
let position = "Default position";
if (!is_displayr)
position = position_options[selectOne("Would you like the NETs to be placed at the beginning (top or left), at the end (bottom or right), or leave them in the default position?", position_options, null, 2)];
// Create the NETs
let cant_modify = [];
let new_questions = createKBoxNETsForCreator(question_objects, k, bottom, position, override_name, cant_modify, remove_dk);
// Make a table for each new question, only do this for QScript
if (!is_displayr) {
let new_group_name = "Questions with " + (bottom ? "Bottom " : "Top ") + k + " NETs";
let new_group;
if (new_questions.length > 0) {
new_group = generateGroupOfSummaryTables(new_group_name, new_questions);
log("Questions with new NET categories have been added to the folder: " + new_group_name);
} else {
new_group = project.report.appendGroup()
new_group.name = new_group_name;
}
if (cant_modify.length > 0) {
let group_name = "Could not create NETs";
log("Some questions could not be modified because the required categories are missing from the table. These have been added to the folder: " + group_name + ". To modify the questions you must first right-click on the table and select Revert.");
generateSubgroupOfSummaryTables(group_name, new_group, cant_modify);
}
} else {
project.report.setSelectedRaw(new_questions);
}
return true;
}
function getTopOrBottomKRecodedCategories(question, k, bottom, options) {
// Set default legacy options if no options supplied
if (!options)
var options = { excludeDK: false };
var remove_dk = options.excludeDK;
var data_reduction = question.dataReduction;
var value_attributes = question.valueAttributes;
var values_list = getAllUnderlyingValues(question);
if (values_list == null)
throw new UserError("Question selected has no underlying values.");
var values_object = values_list.filter(function (obj) { return obj.array.length == 1; });
values_object = values_object.map(function (obj) { return { value: value_attributes.getValue(obj.array[0]), label: obj.label }; });
values_object = values_object.filter(function (obj) { return !isNaN(obj.value); });
if (remove_dk)
values_object = values_object.filter(function (obj) { return !isDontKnow(obj.label); });
values_object.sort(function (a,b) {
return bottom ? a.value - b.value : b.value - a.value;
});
var k_labels = values_object.slice(0,k).map(function (obj) { return obj.label; });
return k_labels;
}
// Function for creating top and bottom box variables. Provides
// the user prompts and creates variables.
//
// - k is the number of categories to combine. If you want the
// user to enter this number, set k as null.
// - is_bottom: set to true for bottom boxes and false for top boxes
function createTopOrBottomBoxVariables(k, is_bottom) {
includeWeb('QScript Utility Functions');
includeWeb('QScript Questionnaire Functions');
includeWeb('QScript Selection Functions');
includeWeb('QScript Value Attributes Functions');
includeWeb('QScript Functions to Generate Outputs');
includeWeb('QScript Data Reduction Functions');
includeWeb('QScript Functions for Processing Arrays');
includeWeb('JavaScript Utilities');
const web_mode = inDisplayr();
const structure_name = web_mode ? "variable sets" : "questions";
// Prompt the user to specify k if not already specified.
if (k == null) {
k = -1;
while (!isPositiveInteger(k)) {
k = prompt("How many categories do you wish to combine? For example, to combine the " + (is_bottom ? "bottom" : "top") + " two categories type '2' and press OK.");
if (!isPositiveInteger(k))
alert(k + "is not a valid number of categories to combine.");
}
}
// Check for already selected questions
const allowed_types = ["Nominal", "Nominal - Multi",
"Ordinal", "Ordinal - Multi"];
const user_selections = getAllUserSelections();
let selected_questions = user_selections.selected_questions;
let sorted_selection = splitArrayIntoApplicableAndNotApplicable(selected_questions, function (q) { return allowed_types.indexOf(q.variableSetStructure) != -1 && !q.isBanner; });
let relevant_questions = sorted_selection.applicable;
selected_questions = relevant_questions;
let selected_datafiles;
if (relevant_questions.length == 0)
{
// Do not prompt in Displayr.
if (web_mode) {
log("None of the selected variable sets are appropriate. Select one or more Nominal/Ordinal or Nominal/Ordinal - Multi variable sets.");
return false;
}
// In Q, guide the user with dialogues
selected_datafiles = dataFileSelection();
if (askYesNo("Would you like to exclude questions that do not look like scales?"))
relevant_questions = getAllScaleQuestions(selected_datafiles);
else
relevant_questions = getAllQuestionsByTypes(selected_datafiles, ["Pick One", "Pick One - Multi"]);
relevant_questions = relevant_questions.filter(function(q) {
let unique_values = q.uniqueValues;
let value_attributes = q.valueAttributes;
let values_to_keep = unique_values.filter(function (x) {
return !isNaN(x) && !value_attributes.getIsMissingData(x) &&
!isDontKnow(value_attributes.getLabel(x));})
return values_to_keep.length > k;
} );
if (!relevant_questions || relevant_questions.length == 0)
{
log("No appropriate questions found.");
return false;
}
question_label_strings = relevant_questions.map(function (q) {
let highest_and_lowest_label = getHighestAndLowestValueAndLabel(q);
return truncateStringForSelectionWindow(q.name) + " ("
+ highest_and_lowest_label.lowest + " ... "
+ highest_and_lowest_label.highest + ")";
})
let selected_indices = selectMany("Please choose which questions you want to create new variables for:\r\n(highest and lowest value labels are shown in brackets)", question_label_strings);
selected_questions = getElementsOfArrayBySelectedIndices(relevant_questions, selected_indices);
}
if (!selected_questions || selected_questions.length == 0)
return false;
selected_datafiles = getDataFileFromQuestions(selected_questions);
if (!selected_datafiles)
return false;
// Check selected questions for 'Dont Know' options
let question_objects = selected_questions.map(function (q) {
return { question: q,
hasDK: nonMissingValueLabels(q).filter(isDontKnow).length > 0 }
});
let questions_containing_dk = question_objects.filter(function (obj) { return obj.hasDK; })
.map(function (obj) { return obj.question; });
// Prompt the user regarding handling of "Don't Know" options.
// This is still important for Displayr users, so keeping it in.
const dk_message = correctTerminology("Some of the selected questions contain 'Don't Know' categories. These categories will not be used to create the "
+ (is_bottom ? "bottom" : "top") + " " + k + " boxes. Do you want to set the 'Don't Know'"
+ " categories as Missing Data (remove them from the base sample)?");
let remove_dk = false;
if (question_objects.some(function (obj) { return obj.hasDK; }))
remove_dk = askYesNo(dk_message);
// Generate a new Pick Any question for each input question with the highest or lowest
// specified non-missing categories selected.
let new_questions = question_objects.map(function (obj) {
let q = obj.question;
let new_question;
let new_values = getTopOrBottomKNonMissingValues(q, k, is_bottom, {excludeDK: true});
if (new_values == null)
return null;
new_question = is_bottom ? createBottomBoxQuestion(q, new_values) : createTopBoxQuestion(q, new_values);
// Remove Don't Know options if user has requested
if (remove_dk && obj.hasDK) {
let value_attributes = new_question.valueAttributes;
let unique_values = new_question.uniqueValues;
unique_values.forEach(function (x) {
if (isDontKnow(value_attributes.getLabel(x)))
setIsMissingDataForVariablesInQuestion(new_question, x, true);
});
}
return new_question;
});
new_questions = new_questions.filter(function (q) { return q != null; });
if (new_questions.length > 0) {
// Move new questions to data sets hover button position if
// user initiated this script from the button.
moveQuestionsToHoverButtonIfShown(new_questions);
// For Q, generate the tables
let new_group_name = (is_bottom ? "Bottom " : "Top ") + k + " category variables";
reportNewRQuestion(new_questions, new_group_name);
}
return true;
}
function mergeScales(mergings, merge_message) {
var web_mode = (!!Q.isOnTheWeb && Q.isOnTheWeb());
var all_scale_points = [];
mergings.forEach(function (obj) {
all_scale_points = all_scale_points.concat(obj.values);
});
// Ask the user to choose which data files to use
var selected_datafiles = dataFileSelection();
var relevant_questions;
if (askYesNo("Q will now show you a list of questions to choose from. Would you like Q to show only questions that look like scales?"))
relevant_questions = getAllScaleQuestions(selected_datafiles);
else
relevant_questions = getAllQuestionsByTypes(selected_datafiles, ["Pick One", "Pick One - Multi"]);
relevant_questions = relevant_questions.filter(function(q) {
var unique_values = q.uniqueValues;
var value_attributes = q.valueAttributes;
var values_to_keep = unique_values.filter(function (x) { return !isNaN(x) && !value_attributes.getIsMissingData(x) && !isDontKnow(value_attributes.getLabel(x));})
return values_to_keep.length == all_scale_points.length;
} );
if (relevant_questions.length == 0) {
log("Could not find any questions to use.");
return false;
}
// Generate a list of applicable questions along with the highest and lowest category labels
var num_questions = relevant_questions.length;
var question_label_strings = [];
var highest_and_lowest_label;
for (var j = 0; j < num_questions; j++) {
highest_and_lowest_label = getHighestAndLowestValueAndLabel(relevant_questions[j]);
question_label_strings.push(truncateStringForSelectionWindow(relevant_questions[j].name) + " ("
+ highest_and_lowest_label.lowest + " ... " + highest_and_lowest_label.highest + ")");
}
// Prompt the user to select the questions that they want to use
var selected_indices = selectMany("Please select the questions in which the categories are to be merged " + merge_message + ": \r\n(highest and lowest value labels are shown in brackets)", question_label_strings);
var selected_questions = getElementsOfArrayBySelectedIndices(relevant_questions, selected_indices);
if (selected_questions.length == 0) {
log("No questions selected.");
return false;
}
// Check selected questions for 'Dont Know' options
var question_objects = selected_questions.map(function (q) {
return { question: q,
hasDK: nonMissingValueLabels(q).filter(isDontKnow).length > 0 }
});
var questions_containing_dk = question_objects.filter(function (obj) { return obj.hasDK; })
.map(function (obj) { return obj.question; });
// Prompt and remove don't know options
if (questions_containing_dk.length > 0) {
if (!web_mode){
var keep_going = confirm("Some questions contain 'Don't Know' categories. These categories will be removed from the questions before merging.");
if (!keep_going)
return false;
}
questions_containing_dk.forEach(function (q) {
var value_attributes = q.valueAttributes;
var unique_values = q.uniqueValues;
unique_values.forEach(function (x) {
if (isDontKnow(value_attributes.getLabel(x)))
value_attributes.setIsMissingData(x, true);
});
});
}
// Create the NETs
var box_obj = mergeCategoriesInManyQuestions(selected_questions, mergings);
tabulateMergedQuestions(box_obj.netQuestions, box_obj.invalidQuestions, merge_message);
return true;
}
// Creates a NET in each of the questions in the array selected_questions
function mergeCategoriesInManyQuestions(selected_questions, nets) {
var n_nets = nets.length;
// Add NETs to the data reductions as specified
var current_question;
var current_data_reduction;
var net_labels;
var new_net_questions = [];
var invalid_questions = [];
var destination_label;
// Loop through selected questions, creating NETs where possible
selected_questions.forEach(function (current_question) {
current_data_reduction = current_question.dataReduction;
var invalid = false;
nets.forEach(function (net) {
if (!invalid) {
var net_name = net.name;
var values_for_net = net.values;
net_labels = getLabelsForValues(current_question, values_for_net);
// check that the top/bottom category labels still exist in the data reduction
if (dataReductionContainsLabels(current_question, net_labels)) {
// Merge
if (values_for_net.length > 1)
current_data_reduction.merge(net_labels, net_name);
else
current_data_reduction.rename(net_labels[0], net_name);
} else {
invalid = true; // Flag to put this question in the list of invalids later.
}
}
});
if (invalid)
invalid_questions.push(current_question);
else
new_net_questions.push(current_question);
});
return {netQuestions: new_net_questions, invalidQuestions: invalid_questions};
}
function tabulateMergedQuestions(new_net_questions, invalid_questions, merge_message) {
// Make a table for each new question
var new_group;
var top_group_name = "Merged Questions " + merge_message;
if (new_net_questions.length > 0) {
new_group = generateGroupOfSummaryTables(top_group_name, new_net_questions);
conditionallyEmptyLog("Questions with merged scales " + merge_message+ " have been added to the folder: " + top_group_name);
} else {
new_group = project.report.appendGroup();
new_group.name = top_group_name;
}
if (invalid_questions.length > 0) {
var invalid_group_name = "Questions with problematic merges";
generateSubgroupOfSummaryTables(invalid_group_name, new_group, invalid_questions);
log("The categories of some questions were not able to be merged. This is most likely due to labels being changed on the table. These questions are shown in the folder: " + invalid_group_name);
}
}
// Main function for facilitating the user's merging of
// customized groups of categories across multiple Pick One
// or Pick One - Multi questions.
function applyCustomMerges(create_new_pick_any_question) {
includeWeb('QScript Utility Functions');
includeWeb('QScript Questionnaire Functions');
includeWeb('QScript Selection Functions');
includeWeb('QScript Value Attributes Functions');
includeWeb('QScript Functions to Generate Outputs');
includeWeb('QScript Data Reduction Functions');
includeWeb('QScript Functions for Processing Arrays');
includeWeb('JavaScript Utilities');
const web_mode = inDisplayr();
let selected_questions = selectQuestionsForCustomMerging(create_new_pick_any_question);
if (!checkQuestionsForCategoryMergingConsistency(selected_questions))
return false;
// At this point, the check for consistent Don't Know
// category values and for consistent unique values,
// has passed, so can just work using the unique
// values of the first selected question.
let first_question = selected_questions[0];
let dk_values = getDKUniqueValues(first_question);
let first_nm_values = getNonMissingUniqueValues(first_question);
let remove_dks = false;
if (dk_values.length > 0)
remove_dks = askYesNo(correctTerminology("The selected questions contain 'Don't Know' options. Would you like to remove them (i.e. set them as Missing Data) before continuing?"));
// Don't include DK values in user prompts if the user has elected to
// remove them
if (remove_dks)
first_nm_values = first_nm_values.filter(x => dk_values.indexOf(x) == -1)
let value_attributes = first_question.valueAttributes;
let prompt_objects = first_nm_values.map(function (x) {
return {value: x, label: `${x} (${value_attributes.getLabel(x)})` }
})
let mergings = promptUserForCustomMergings(prompt_objects);
if (!checkUserEnteredMergingsAreValid(mergings))
return false;
// Must be done after user prompts are completed as Displayr
// is not allowed to prompt the user after modifying the
// document.
if (remove_dks)
removeDontKnowCategories(selected_questions, dk_values);
applyUserSpecifiedCustomMergings(selected_questions, mergings, create_new_pick_any_question)
return true;
}
function selectQuestionsForCustomMerging(create_new_pick_any_question) {
// In Displayr, take selections from Data Sets tree.
// In Q, guide the user selection with prompts, showing
// them which sets of questions are likely to be
// consistent with one another.
if (inDisplayr()) {
let user_selections = getAllUserSelections();
return user_selections.selected_questions;
}
let selected_datafiles = dataFileSelection();
let relevant_questions = getAllQuestionsByTypes(selected_datafiles, ["Pick One", "Pick One - Multi"]);
// Count scale points for each question so we can offer the user
// the choice of which number of scale points to apply to.
let scale_object = relevant_questions.map(function (question) {
let non_missing_values = getNonMissingUniqueValues(question);
return { question: question, points: non_missing_values.length };
});
scale_object = scale_object.filter(function (obj) { return obj.points > 2; });
let num_scale_points = uniqueElementsInArray(scale_object.map(function (obj) { return obj.points; }));
num_scale_points.sort(function (a,b) { return a-b; });
let explanation_message = "This QScript allows you to merge categories in many questions at once.\r\n\r\n"
+"You first need to specify how many categories are contained in the questions that you want to modify.\r\n\r\n"
+"For example, if you would like to modify questions with 5 categories, you would choose 5 in the list below.\r\n\r\n"
+"Scales with these numbers of categories have been identified in your data:";
if (create_new_pick_any_question) {
explanation_message = "This QScript allows you to create new variables from the merged categories of many questions at once.\r\n\r\n"
+"You first need to specify how many categories are contained the questions you want to work with.\r\n\r\n"
+"For example, if you want to create new variables by merging categories from questions which have 5 categories, you would choose 5 in the list below.\r\n\r\n"
+"Scales with these numbers of categories have been identified in your data:";
}
let selected_num_points = num_scale_points[selectOne(explanation_message, num_scale_points)];
relevant_questions = scale_object.filter(function (obj) { return obj.points == selected_num_points}).map(function (obj) { return obj.question});
let question_label_strings = relevant_questions.map(function (q) {
let highest_and_lowest_label = getHighestAndLowestValueAndLabel(q);
return truncateStringForSelectionWindow(q.name) + " ("
+ highest_and_lowest_label.lowest + " ... "
+ highest_and_lowest_label.highest + ")";
});
let selected_indices = selectMany("The following questions have " + selected_num_points + " categories. Choose the questions whose categories you want to merge:\r\n(highest and lowest value labels are shown in brackets)", question_label_strings);
let selected_questions = getElementsOfArrayBySelectedIndices(relevant_questions, selected_indices);
return selected_questions;
}
// Return all values whose labels look like "Don't Know" options
function getDKUniqueValues(question) {
let first_value_attributes = question.valueAttributes;
return question.uniqueValues.filter(function (x) { return !isNaN(x) && !first_value_attributes.getIsMissingData(x) && isDontKnow(first_value_attributes.getLabel(x)); });
}
// Format information about entered merging for message boxes
function formatCustomMergeString(obj) {
return obj.name + ": [" + obj.values.join(", ") + "]";
}
// Ensure that the selected questions have identical sets of
// non-missing unique values. Otherwise, we are not guaranteed
// to be able to merge categories consistently across the questions.
function checkQuestionsForCategoryMergingConsistency(selected_questions) {
if (selected_questions.length == 0) {
log(correctTerminology("No question selected."))
return false;
}
if (selected_questions.some(x => ["Pick One", "Pick One - Multi"].indexOf(x.questionType) == -1)) {
log(correctTerminology("Some of the selected questions are not Pick One or Pick One - Multi questions. Custom mergings cannot be applied to these."));
return false;
}
// Properties of first question are used to check consistency.
let first_question = selected_questions[0];
let first_dk_values = getDKUniqueValues(first_question);
let first_nm_values = getNonMissingUniqueValues(first_question);
let values_properties = selected_questions.map(function(q) {
return {question: q, values: getNonMissingUniqueValues(q)};
})
let first_inconsistent;
let values_inconsistent = values_properties.some(function (obj) {
if (obj.question.equals(first_question))
return false;
let current_values = obj.values;
if (current_values.length != first_nm_values.length
|| current_values.some(function (x) { return first_nm_values.indexOf(x) == -1; } )
|| first_nm_values.some(function (x) { return current_values.indexOf(x) == -1; } ) ) {
first_inconsistent = obj;
return true;
}
return false;
});
if (values_inconsistent) {
let first_inconsistent_question = first_inconsistent.question;
let first_inconsistent_values = first_inconsistent.values;
let first_but_not_second = difference(first_nm_values, first_inconsistent_values);
let second_but_not_first = difference(first_inconsistent_values, first_nm_values);
log(correctTerminology("The selected questions have different sets of source values. Their categories can't be merged consistently."));
if (first_but_not_second.length > 0) {
log(`The value(s): ${first_but_not_second.join(", ")} are present in '${first_question.name}' but are not present in '${first_inconsistent_question.name}'.`);
}
if (second_but_not_first.length > 0) {
log(`The value(s): ${second_but_not_first.join(", ")} are present in '${first_inconsistent_question.name}' but are not present in '${first_question.name}'.`);
}
return false;
}
let dk_inconsistent = selected_questions.some(function (question) {
let dk_values = getDKUniqueValues(question);
if (dk_values.length != first_dk_values.length)
return true;
else
return false;
})
if (dk_inconsistent) {
log(correctTerminology("Some of the selected questions appear to have 'Don't know' options while others do not. Their categories can't be merged consistently."));
return false;
}
return true;
}
function removeDontKnowCategories(selected_questions, dk_values) {
selected_questions.forEach(function (question) {
let value_attributes = question.valueAttributes;
dk_values.forEach(function (x) {
value_attributes.setIsMissingData(x, true);
});
});
}
function promptUserForCustomMergings(prompt_objects) {
// Cycle through prompts for mergings
let remaining_values = prompt_objects.map(x => x.value);
let mergings = [];
let current_merging_text = "";
while (remaining_values.length > 0) {
let remaining_labels_strings = prompt_objects.filter(x => remaining_values.indexOf(x.value) > -1).map(x => x.label);
let selected_indices = selectMany("Select the values to merge. If you have finished, select no values and click OK.", remaining_labels_strings);
let selected_values = getElementsOfArrayBySelectedIndices(remaining_values, selected_indices);
if (selected_values.length == 0) // User has finished
break;
remaining_values = difference(remaining_values, selected_values);
let label_prompt_text = `Enter a label for the category with values (${selected_values.join(", ")})`;
let entered_label = "";
while (entered_label == "")
entered_label = prompt(current_merging_text + label_prompt_text);
mergings.push({ name: entered_label, values: selected_values });
}
return mergings;
}
function checkUserEnteredMergingsAreValid(mergings) {
// No Duplicate Codes Allowed
let all_values = [];
mergings.map(function (obj) { return obj.values; }).forEach(function (a) { all_values = all_values.concat(a); });
if (arrayHasDuplicateElements(all_values)) {
log("Some of the entered values are repeated - each value must appear in a single merged category.");
return false;
}
return true;
}
function applyUserSpecifiedCustomMergings(selected_questions, mergings, create_new_pick_any_question) {
const web_mode = inDisplayr();
// Label to use when presenting tables with merged categories (Q only)
let merge_message = "(" + mergings.map(function (obj) { return obj.name; }).join(", ") + ")"
// Perform mergings
if (create_new_pick_any_question) {
let net_questions = [];
selected_questions.forEach(function (question) {
mergings.forEach(function (obj) {
let new_name = preventDuplicateQuestionName(question.dataFile, question.name + " " + obj.name);
let new_question = question.duplicate(new_name);
new_question.questionType = "Pick Any";
let v = new_question.variables;
if (v.length === 1)
v[0].label = new_question.name;
new_question.uniqueValues.forEach(function (x) {
if (obj.values.indexOf(x) > -1)
setCountThisValueForVariablesInQuestion(new_question, x, true);
else
setCountThisValueForVariablesInQuestion(new_question, x, false);
});
new_question.needsCheckValuesToCount = false;
insertAtHoverButtonIfShown(new_question);
net_questions.push(new_question);
});
});
if (!web_mode)
tabulateMergedQuestions(net_questions, [], merge_message);
} else {
let merge_object = mergeCategoriesInManyQuestions(selected_questions, mergings);
if (!web_mode)
tabulateMergedQuestions(merge_object.netQuestions, merge_object.invalidQuestions, merge_message);
}
}
// Interpret comma-separated values entered by user
function convertCommaSeparatedStringToValues(string) {
let split_vals = string.split(",");
split_vals = split_vals.map(function (x) { return parseFloat(x); });
return split_vals;
}
// Return all unique values which are not NaN and have not been set as missing
function getNonMissingUniqueValues(question) {
var value_attributes = question.valueAttributes;
var non_missing_values = question.uniqueValues.filter(function (x) {
return !isNaN(x) && !value_attributes.getIsMissingData(x);
});
return non_missing_values;
}