Grouper ABAC Crashplan deprovisioning example

CREATE TABLE public.penn_crashplan_user (
	user_id int8 NOT NULL,
	user_uid varchar(100) NOT NULL,
	status_active varchar(1) NOT NULL DEFAULT 'F'::character varying,
	status_blocked varchar(1) NOT NULL DEFAULT 'F'::character varying,
	username varchar(100) NULL,
	org_name varchar(200) NULL,
	user_active varchar(1) NOT NULL DEFAULT 'F'::character varying,
	"admin" varchar(1) NOT NULL DEFAULT 'F'::character varying,
	penn_id varchar(100) NULL,
	external_subject_name varchar(1024) NULL,
	CONSTRAINT penn_crashplan_user_pk PRIMARY KEY (user_id)
);
CREATE INDEX penn_crashplan_user_penn_id_idx ON public.penn_crashplan_user USING btree (penn_id);

CREATE TABLE public.penn_crashplan_role (
	user_id int8 NOT NULL,
	role_name varchar(200) NOT NULL,
	CONSTRAINT penn_crashplan_role_pk PRIMARY KEY (user_id, role_name)
);

ALTER TABLE public.penn_crashplan_role ADD CONSTRAINT penn_crashplan_role_fk FOREIGN KEY (user_id) REFERENCES public.penn_crashplan_user(user_id) ON DELETE CASCADE;

import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang.StringUtils;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;

import edu.internet2.middleware.grouper.Group;
import edu.internet2.middleware.grouper.GroupFinder;
import edu.internet2.middleware.grouper.GroupSave;
import edu.internet2.middleware.grouper.GrouperSession;
import edu.internet2.middleware.grouper.Stem;
import edu.internet2.middleware.grouper.StemFinder;
import edu.internet2.middleware.grouper.app.loader.GrouperLoader;
import edu.internet2.middleware.grouper.app.loader.OtherJobScript;
import edu.internet2.middleware.grouper.cfg.GrouperConfig;
import edu.internet2.middleware.grouper.exception.GrouperSessionException;
import edu.internet2.middleware.grouper.group.TypeOfGroup;
import edu.internet2.middleware.grouper.misc.GrouperSessionHandler;
import edu.internet2.middleware.grouper.util.GrouperHttpClient;
import edu.internet2.middleware.grouper.util.GrouperHttpMethod;
import edu.internet2.middleware.grouper.util.GrouperUtil;
import edu.internet2.middleware.grouperClient.jdbc.GcDbAccess;
import edu.internet2.middleware.grouperClient.jdbc.tableSync.GcTableSyncFromData;
import edu.internet2.middleware.grouperClient.util.GrouperClientUtils;

//uncomment to compile in eclipse (and last line)
//public class Test58crashplan {
  
  class TheState {
    Map<String, Object> debugMap = new LinkedHashMap<String, Object>();
    //  String crashPlanClientId = "key-028abc-xyz-e09d8e";
    //  String crashPlanClientSecret = "abc";
    //  String crashPlanUrl = "https://console.us2.crashplan.com";
    String crashPlanClientId = GrouperConfig.retrieveConfig().propertyValueString("crashPlanClientId");
    String crashPlanClientSecret = GrouperConfig.retrieveConfig().propertyValueString("crashPlanClientSecret");
    String crashPlanUrl = GrouperUtil.stripLastSlashIfExists(GrouperConfig.retrieveConfig().propertyValueString("crashPlanUrl"));

    String crashPlanLocalEntityFolderName = "penn:isc:ait:apps:crashPlan:service:emailsNotMatched";
    String crashPlanLoaderGroupNameOrg = "penn:isc:ait:apps:crashPlan:service:loader:orgLoader";
    String crashPlanLoaderGroupNameRole = "penn:isc:ait:apps:crashPlan:service:loader:roleLoader";
    String crashPlanLoaderGroupNameStatus = "penn:isc:ait:apps:crashPlan:service:loader:statusLoader";
    String crashPlanLoaderGroupNameAdmin = "penn:isc:ait:apps:crashPlan:service:loader:adminLoader";
 
    String crashPlanDeactiveUserGroupName = "penn:isc:ait:apps:crashPlan:service:policy:deactivateUsers";
    String crashPlanBlockUserGroupName = "penn:isc:ait:apps:crashPlan:service:policy:blockUsers";
    
    Stem crashPlanLocalEntityFolder = null;
    
    int crashPlanTokenExpireMinutes = GrouperConfig.retrieveConfig().propertyValueInt("crashPlanTokenExpireMinutes", 10);
    int crashPlanPageSize = GrouperConfig.retrieveConfig().propertyValueInt("crashPlanPageSize", 1000);

    // admins, whether they are active or not
    Set<BigDecimal> userIdsAdmins = new HashSet<>();
    

    String currentToken = null;
    long currentTokenRetrieved = -1;

    int USER_ID_INDEX = 0;
    int USER_UID_INDEX = 1;
    int STATUS_ACTIVE_INDEX = 2;
    int STATUS_BLOCKED_INDEX = 3;
    int USERNAME_INDEX = 4;
    int ORG_NAME_INDEX = 5;
    int ADMIN_INDEX = 6;
    int PENN_ID_INDEX = 7;
    int EXTERNAL_SUBJECT_NAME_INDEX = 8;
    
    List<Object[]> wsRows = new ArrayList<>();
    List<String> userNames = new ArrayList<String>();
    
    Map<BigDecimal, Set<String>> userIdToRoleNames = new HashMap<>();
    
    GrouperSession grouperSession = null;

  }
  
  String retrieveAccessToken(TheState theState) {

    if (theState.currentToken != null && (System.currentTimeMillis() - theState.currentTokenRetrieved) / (1000 * 60) < theState.crashPlanTokenExpireMinutes) {
      GrouperUtil.mapAddValue(theState.debugMap, "accessTokenCache", 1);
      return theState.currentToken;
    }
    GrouperUtil.mapAddValue(theState.debugMap, "accessTokenRetrieve", 1);
    GrouperHttpClient grouperHttpClient = new GrouperHttpClient().addHeader("Accept", "application/json").
      addHeader("Content-Type", "application/json").assignUser(theState.crashPlanClientId).assignPassword(theState.crashPlanClientSecret).
      assignUrl(theState.crashPlanUrl + "/api/v3/oauth/token?grant_type=client_credentials").executeRequest();
    //curl -X POST -k -H 'Accept: application/json' -H 'Content-Type: application/json' -H 'Authorization: Basic abc:xyz' 
    // -i 'https://console.us2.crashplan.com/api/v3/oauth/token?grant_type=client_credentials'

    String responseBody = grouperHttpClient.getResponseBody();
    if (grouperHttpClient.getResponseCode() != 200) {
      throw new RuntimeException("Response code for retrieveToken: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
    }
    JsonNode jsonJacksonNode = GrouperUtil.jsonJacksonNode(responseBody);
    String accessToken = GrouperUtil.jsonJacksonGetString(jsonJacksonNode, "access_token");
    GrouperUtil.assertion(!StringUtils.isBlank(accessToken), "Token is blank!");
    theState.currentToken = accessToken;
    theState.currentTokenRetrieved = System.currentTimeMillis();
    return theState.currentToken;
  }
  

  void retrieveAdmins(TheState theState) {
    
    int timeToLive=100000/theState.crashPlanPageSize;
    int pgNum = 1;
    while (true) {
      
      // dont endless loop
      if (timeToLive-- < 0) {
        throw new RuntimeException("Endless loop");
      }        

      // get an access token each time so it isnt expired
      String accessToken = retrieveAccessToken(theState);
      
      // make the call
      GrouperHttpClient grouperHttpClient = new GrouperHttpClient().assignGrouperHttpMethod(GrouperHttpMethod.get).addHeader("Accept", "application/json").
          addHeader("Content-Type", "application/json").addHeader("Authorization", "Bearer " + accessToken).
          assignUrl(theState.crashPlanUrl + "/api/v1/User?admins=true&pgSize=" + theState.crashPlanPageSize + "&pgNum=" + pgNum).executeRequest();
      
      // make sure valid response
      String responseBody = grouperHttpClient.getResponseBody();
      if (grouperHttpClient.getResponseCode() != 200) {
        throw new RuntimeException("Response code for retrieve admins: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
      }
      //{"metadata":{"timestamp":"2023-09-15T01:45:56.460Z","params":{"pgNum":"6","pgSize":"1000"}},"data":{"totalCount":4972,"users":[]}}

      // make sure server supports page size
      JsonNode jsonJacksonNode = GrouperUtil.jsonJacksonNode(responseBody);
      int pageSizeFromResponse = GrouperUtil.jsonJacksonGetIntegerFromJsonPointer(jsonJacksonNode, "/metadata/params/pgSize");
      GrouperUtil.assertion(pageSizeFromResponse == theState.crashPlanPageSize, "Page size different " + pageSizeFromResponse + " != " + theState.crashPlanPageSize);

      // get the users
      JsonNode usersNodeJsonNode = GrouperUtil.jsonJacksonGetNodeFromJsonPointer(jsonJacksonNode, "/data/users");

      // if no users we are done
      if (usersNodeJsonNode == null) {
        break;
      }
      ArrayNode usersNode = (ArrayNode)usersNodeJsonNode;
      if (usersNode.size() == 0) {
        break;
      }
      for (int i=0;i<usersNode.size();i++) {
        //  {
        //    "userId":13224985,
        JsonNode userNode = usersNode.get(i);
        BigDecimal userId = new BigDecimal(GrouperUtil.jsonJacksonGetLong(userNode, "userId"));
        theState.userIdsAdmins.add(userId);
      }  
      pgNum++;
    }
    theState.debugMap.put("adminsRetrieve", GrouperUtil.length(theState.userIdsAdmins));
  }
  
  void retrieveUsers(TheState theState) {
    
    int timeToLive=100000/theState.crashPlanPageSize;
    int pgNum = 1;
    while (true) {
      
      // dont endless loop
      if (timeToLive-- < 0) {
        throw new RuntimeException("Endless loop");
      }        

      // get an access token each time so it isnt expired
      String accessToken = retrieveAccessToken(theState);
      
      // make the call
      GrouperHttpClient grouperHttpClient = new GrouperHttpClient().assignGrouperHttpMethod(GrouperHttpMethod.get).addHeader("Accept", "application/json").
          addHeader("Content-Type", "application/json").addHeader("Authorization", "Bearer " + accessToken).
          assignUrl(theState.crashPlanUrl + "/api/v1/User?incRoles=true&pgSize=" + theState.crashPlanPageSize + "&pgNum=" + pgNum).executeRequest();
      
      // make sure valid response
      String responseBody = grouperHttpClient.getResponseBody();
      if (grouperHttpClient.getResponseCode() != 200) {
        throw new RuntimeException("Response code: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
      }
      //{"metadata":{"timestamp":"2023-09-15T01:45:56.460Z","params":{"pgNum":"6","pgSize":"1000"}},"data":{"totalCount":4972,"users":[]}}

      // make sure server supports page size
      JsonNode jsonJacksonNode = GrouperUtil.jsonJacksonNode(responseBody);
      int pageSizeFromResponse = GrouperUtil.jsonJacksonGetIntegerFromJsonPointer(jsonJacksonNode, "/metadata/params/pgSize");
      GrouperUtil.assertion(pageSizeFromResponse == theState.crashPlanPageSize, "Page size different " + pageSizeFromResponse + " != " + theState.crashPlanPageSize);

      // get the users
      JsonNode usersNodeJsonNode = GrouperUtil.jsonJacksonGetNodeFromJsonPointer(jsonJacksonNode, "/data/users");

      // if no users we are done
      if (usersNodeJsonNode == null) {
        break;
      }
      ArrayNode usersNode = (ArrayNode)usersNodeJsonNode;
      if (usersNode.size() == 0) {
        break;
      }
      for (int i=0;i<usersNode.size();i++) {
        JsonNode userNode = usersNode.get(i);
        
        Object[] wsRow = new Object[9];
        theState.wsRows.add(wsRow);
                
        //  int USER_ID_INDEX = 0;
        // {
        //  "userId":13224985,
        
        BigDecimal userId = new BigDecimal(GrouperUtil.jsonJacksonGetLong(userNode, "userId"));
        wsRow[theState.USER_ID_INDEX] = userId;
        
        //  int USER_UID_INDEX = 1;
        //  "userUid":"933696115269786900",
        
        wsRow[theState.USER_UID_INDEX] = GrouperUtil.jsonJacksonGetString(userNode, "userUid");

        // int STATUS_ACTIVE_INDEX = 2;
        //  "active":true,
        wsRow[theState.STATUS_ACTIVE_INDEX] = GrouperUtil.jsonJacksonGetBoolean(userNode, "active") ? "T" : "F";

        //  int STATUS_BLOCKED_INDEX = 3;
        //  "blocked":false,
        wsRow[theState.STATUS_BLOCKED_INDEX] = GrouperUtil.jsonJacksonGetBoolean(userNode, "blocked") ? "T" : "F";
        
         //  int USERNAME_INDEX = 4;
        //  "username":"{60b78e88-ead8-445c-9cfd-0b87f74ea6cd}@wharton.upenn.edu",
        wsRow[theState.USERNAME_INDEX] = GrouperUtil.jsonJacksonGetString(userNode, "username");
        theState.userNames.add((String)wsRow[theState.USERNAME_INDEX]);
        
        //  "orgName":"CTS-Client Support",
        //  int ORG_NAME_INDEX = 5;
        wsRow[theState.ORG_NAME_INDEX] = GrouperUtil.jsonJacksonGetString(userNode, "orgName");

        //  int ADMIN_INDEX = 6;
        //  "roles":[
        //           "org-admin",
        //           "desktop-user",
        //           "proe-user"
        //        ],
        Set<String> roles = GrouperUtil.nonNull(GrouperUtil.jsonJacksonGetStringSet(userNode, "roles"));
        boolean isAdmin = theState.userIdsAdmins.contains(wsRow[theState.USER_ID_INDEX]);
        for (String role : GrouperUtil.nonNull(roles)) {
          if (role.toLowerCase().contains("admin")) {
            isAdmin=true;
          }
        }
        wsRow[theState.ADMIN_INDEX] = isAdmin ? "T" : "F";
        
        theState.userIdToRoleNames.put(userId, roles);
        
      }
      pgNum++;
    }
  }
  
  public void retrievePennids(TheState theState) {
    
    Map<String, String> pennkeyToPennid = new HashMap<String, String>();
    
    List<Object[]> wsRowsFromPenn = new ArrayList<>();
    for (Object[] wsRow : theState.wsRows) {
      String username = (String)wsRow[theState.USERNAME_INDEX];
      if (username.contains("@") && (username.endsWith("@upenn.edu") || username.endsWith(".upenn.edu") ))  {
        wsRowsFromPenn.add(wsRow);
      }
    }
    
    int batchSize = 1000;
    int numberOfBatches = GrouperUtil.batchNumberOfBatches(wsRowsFromPenn, batchSize, false);
    for (int i=0;i<numberOfBatches;i++) {

      List<Object[]> batchOfRows = GrouperUtil.batchList(wsRowsFromPenn, batchSize, i);
      
      String sql = "select subject_id, subject_identifier0 from grouper_members where subject_source = 'pennperson' and subject_identifier0 in (" +
          GrouperClientUtils.appendQuestions(GrouperUtil.length(batchOfRows)) + ")";
    
      GcDbAccess gcDbAccess = new GcDbAccess().sql(sql);
            
      for (Object[] wsRow : batchOfRows) {
        String username = (String)wsRow[theState.USERNAME_INDEX];
        String pennkey = GrouperUtil.prefixOrSuffix(username, "@", true);
        gcDbAccess.addBindVar(pennkey);
      }
      
      List<Object[]> pennIdPennkeys = gcDbAccess.selectList(Object[].class);
      for (Object[] pennIdPennkey : GrouperUtil.nonNull(pennIdPennkeys)) {
        String pennId = (String)pennIdPennkey[0];
        String pennkey = (String)pennIdPennkey[1];
        pennkeyToPennid.put(pennkey, pennId);
      }
    }
    theState.debugMap.put("pennIdsFound", pennkeyToPennid.size());
    for (Object[] wsRow : wsRowsFromPenn) {
      String username = (String)wsRow[theState.USERNAME_INDEX];
      String pennkey = GrouperUtil.prefixOrSuffix(username, "@", true);
      String pennId = pennkeyToPennid.get(pennkey);
      if (!StringUtils.isBlank(pennId)) {
        wsRow[theState.PENN_ID_INDEX] = pennId;
      }
    }
  }

  public void retrieveExistingLocalEntities(TheState theState) {
    theState.crashPlanLocalEntityFolder = StemFinder.findByName(theState.grouperSession, theState.crashPlanLocalEntityFolderName, true);
    Map<String, String> displayExtensionToName = new HashMap<>();
    List<Object[]> nameDisplayExtensions = new GcDbAccess().
      sql("select name, display_extension from grouper_groups gg where gg.type_of_group = 'entity' and gg.parent_stem = ?").
      addBindVar(theState.crashPlanLocalEntityFolder.getId()).selectList(Object[].class);
    
    // put existing local entities in a map
    for (Object[] nameDisplayExtension : nameDisplayExtensions) {
      String name = (String)nameDisplayExtension[0];
      String displayExtension = (String)nameDisplayExtension[1];
      displayExtensionToName.put(displayExtension, name);
    }
    theState.debugMap.put("existingLocalEntities", displayExtensionToName.size());
    int matchedRows = 0;
    
    // see which accounts match
    for (Object[] wsRow : theState.wsRows) {
      String pennId = (String)wsRow[theState.PENN_ID_INDEX];
      if (!StringUtils.isBlank(pennId)) {
        continue;
      }
      String email = (String)wsRow[theState.USERNAME_INDEX];
      String name = displayExtensionToName.get(email);
      if (!StringUtils.isBlank(name)) {
        wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX] = name;
        matchedRows++;
      }
    }
    theState.debugMap.put("existingLocalEntityMatches", matchedRows);
    
    // remove matches
    for (Object[] wsRow : theState.wsRows) {
      String email = (String)wsRow[theState.USERNAME_INDEX];
      String externalSubjectName = (String)wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX];
      if (!StringUtils.isBlank(externalSubjectName)) {
        displayExtensionToName.remove(email);
      }
    }
    
    //delete unused
    theState.debugMap.put("deletedLocalEntities", displayExtensionToName.size());

    int logSize = 100;
    for (String externalSubjectName : displayExtensionToName.values()) {
      Group localEntity = GroupFinder.findByName(externalSubjectName, true);
      localEntity.delete();
      if (OtherJobScript.retrieveFromThreadLocal() != null) {
        OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addDeleteCount(1);
      }

      if (logSize-- > 0) {
        theState.debugMap.put("deleted_" + GrouperUtil.extensionFromName(externalSubjectName), true);
      }
    }
    
  }

  public void createMissingLocalEntities(TheState theState) {
        
    int newLocalEntities = 0;
    
    // see which accounts need a local entity
    for (Object[] wsRow : theState.wsRows) {
      String email = (String)wsRow[theState.USERNAME_INDEX];
      String pennId = (String)wsRow[theState.PENN_ID_INDEX];
      String externalSubjectName = (String)wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX];
      if (StringUtils.isBlank(pennId) && StringUtils.isBlank(externalSubjectName)) {
        
        String extension = email.replaceAll("[^a-zA-Z0-9_-]", "_");
        
        Group group = new GroupSave().assignName(theState.crashPlanLocalEntityFolderName + ":" + extension).
          assignDisplayExtension(email).assignTypeOfGroup(TypeOfGroup.entity).save();
        
        if (OtherJobScript.retrieveFromThreadLocal() != null) {
          OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addInsertCount(1);
        }

        wsRow[theState.EXTERNAL_SUBJECT_NAME_INDEX] = group.getName();
        newLocalEntities++;
      }
    }
    theState.debugMap.put("localEntitesCreated", newLocalEntities);
        
  }

  public void syncWsRows(TheState theState) {
    
    List<String> columnNames = GrouperUtil.toList("user_id", "user_uid", "status_active", "status_blocked", 
        "username", "org_name", "admin", "penn_id", "external_subject_name");

    List<String> columnNamesPrimaryKey = GrouperUtil.toList("user_id");
  
    new GcTableSyncFromData().assignDebugMap(theState.debugMap).assignDebugMapPrefix("user_").assignConnectionName("grouper").assignTableName("penn_crashplan_user")
      .assignColumnNames(columnNames).assignColumnNamesPrimaryKey(columnNamesPrimaryKey).assignData(theState.wsRows).sync();

    if (OtherJobScript.retrieveFromThreadLocal() != null) {
      OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addTotalCount(GrouperUtil.length(theState.wsRows));
    }
    
    columnNames = GrouperUtil.toList("user_id", "role_name");

    columnNamesPrimaryKey = GrouperUtil.toList("user_id, role_name");
  
    List<Object[]> userIdRoleNames = new ArrayList<>();

    for (BigDecimal userId : theState.userIdToRoleNames.keySet()) {
      Set<String> roles = theState.userIdToRoleNames.get(userId);
      for (String role : roles) {
        userIdRoleNames.add(GrouperUtil.toArrayObject(userId, role));
      }
    }
        
    new GcTableSyncFromData().assignDebugMap(theState.debugMap).assignDebugMapPrefix("role_").assignConnectionName("grouper").assignTableName("penn_crashplan_role")
      .assignColumnNames(columnNames).assignColumnNamesPrimaryKey(columnNamesPrimaryKey).assignData(userIdRoleNames).sync();

    if (OtherJobScript.retrieveFromThreadLocal() != null) {
      OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().addTotalCount(GrouperUtil.length(userIdRoleNames));
      OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().
        addInsertCount(GrouperUtil.intValue(theState.debugMap.get("insertsCount"), 0));
      OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().
        addUpdateCount(GrouperUtil.intValue(theState.debugMap.get("updatesCount"), 0));
      OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().
        addDeleteCount(GrouperUtil.intValue(theState.debugMap.get("deletesCount"), 0));
      
    }

  }

  public void runLoaders(TheState theState) {
    
    // these will run in this process since this is a daemon itself and is running on the daemon server
    Group crashPlanLoaderGroupRole = GroupFinder.findByName(theState.crashPlanLoaderGroupNameRole, true);
    GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupRole, false);

    Group crashPlanLoaderGroupAdmin = GroupFinder.findByName(theState.crashPlanLoaderGroupNameAdmin, true);
    GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupAdmin, false);
    
    Group crashPlanLoaderGroupOrg = GroupFinder.findByName(theState.crashPlanLoaderGroupNameOrg, true);
    GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupOrg, false);

    Group crashPlanLoaderGroupStatus = GroupFinder.findByName(theState.crashPlanLoaderGroupNameStatus, true);
    GrouperLoader.runJobOnceForGroup(theState.grouperSession, crashPlanLoaderGroupStatus, false);

  }

  public void blockAndDeactivateUsers(TheState theState) {

    Group usersToBlock = GroupFinder.findByName(theState.crashPlanBlockUserGroupName, true);
    
    Group usersToDeactivate = GroupFinder.findByName(theState.crashPlanDeactiveUserGroupName, true);
    
//    
//      // dont endless loop
//      if (timeToLive-- < 0) {
//        throw new RuntimeException("Endless loop");
//      }        
//
//      // get an access token each time so it isnt expired
//      String accessToken = retrieveAccessToken(theState);
//      
//      // make the call
//      GrouperHttpClient grouperHttpClient = new GrouperHttpClient().assignGrouperHttpMethod(GrouperHttpMethod.get).addHeader("Accept", "application/json").
//          addHeader("Content-Type", "application/json").addHeader("Authorization", "Bearer " + accessToken).
//          assignUrl(theState.crashPlanUrl + "/api/v1/User?incRoles=true&pgSize=" + theState.crashPlanPageSize + "&pgNum=" + pgNum).executeRequest();
//      
//      // make sure valid response
//      String responseBody = grouperHttpClient.getResponseBody();
//      if (grouperHttpClient.getResponseCode() != 200) {
//        throw new RuntimeException("Response code: " + grouperHttpClient.getResponseCode() + ", " + responseBody);
//      }

  }
  
  public void runLogic() {

    final TheState theState = new TheState();
    try {
      GrouperSession.internal_callbackRootGrouperSession(new GrouperSessionHandler() {
        
        @Override
        public Object callback(GrouperSession theGrouperSession) throws GrouperSessionException {
          theState.grouperSession = theGrouperSession;
          retrieveAdmins(theState);
          retrieveUsers(theState);
          retrievePennids(theState);
          retrieveExistingLocalEntities(theState);
          createMissingLocalEntities(theState);
          syncWsRows(theState);
          runLoaders(theState);
          //blockAndDeactivateUsers(theState);
          return null;
        }
      });
    } catch (Exception e) {
      theState.debugMap.put("exception", GrouperUtil.getFullStackTrace(e));
      throw new RuntimeException(e);
    } finally {
      if (OtherJobScript.retrieveFromThreadLocal() != null) {
        OtherJobScript.retrieveFromThreadLocal().getOtherJobInput().getHib3GrouperLoaderLog().appendJobMessage(GrouperUtil.mapToString(theState.debugMap));
      } else {
        System.out.println(GrouperUtil.mapToString(theState.debugMap));
      }
    }
    
  }
  
  runLogic();
  

//  public static void main(String[] args) {
//    new Test58crashplan().runLogic();
//  }
//
//}

grouperDataField.cp_active.fieldAliases = cp_active
grouperDataField.cp_active.fieldDataAssignableTo = individuals
grouperDataField.cp_active.fieldDataSource = provider
grouperDataField.cp_active.fieldDataStorePit = true
grouperDataField.cp_active.fieldDataStorePitDays = 200
grouperDataField.cp_active.fieldDataStructure = rowColumn
grouperDataField.cp_active.fieldDataType = boolean
grouperDataField.cp_active.fieldDataUse = access
grouperDataField.cp_active.fieldPrivacyRealm = crashplan
grouperDataField.cp_blocked.fieldAliases = cp_blocked
grouperDataField.cp_blocked.fieldDataAssignableTo = individuals
grouperDataField.cp_blocked.fieldDataSource = provider
grouperDataField.cp_blocked.fieldDataStorePit = true
grouperDataField.cp_blocked.fieldDataStorePitDays = 200
grouperDataField.cp_blocked.fieldDataStructure = rowColumn
grouperDataField.cp_blocked.fieldDataType = boolean
grouperDataField.cp_blocked.fieldDataUse = access
grouperDataField.cp_blocked.fieldPrivacyRealm = crashplan
grouperDataField.cp_known.fieldAliases = cp_known
grouperDataField.cp_known.fieldDataAssignableTo = individuals
grouperDataField.cp_known.fieldDataSource = provider
grouperDataField.cp_known.fieldDataStorePit = true
grouperDataField.cp_known.fieldDataStorePitDays = 200
grouperDataField.cp_known.fieldDataStructure = rowColumn
grouperDataField.cp_known.fieldDataType = boolean
grouperDataField.cp_known.fieldDataUse = access
grouperDataField.cp_known.fieldPrivacyRealm = crashplan
grouperDataField.cp_org.fieldAliases = cp_org
grouperDataField.cp_org.fieldDataAssignableTo = individuals
grouperDataField.cp_org.fieldDataSource = provider
grouperDataField.cp_org.fieldDataStorePit = true
grouperDataField.cp_org.fieldDataStorePitDays = 200
grouperDataField.cp_org.fieldDataStructure = rowColumn
grouperDataField.cp_org.fieldDataType = string
grouperDataField.cp_org.fieldDataUse = access
grouperDataField.cp_org.fieldPrivacyRealm = crashplan
grouperDataField.cp_role.fieldAliases = cp_role
grouperDataField.cp_role.fieldDataAssignableTo = individuals
grouperDataField.cp_role.fieldDataSource = provider
grouperDataField.cp_role.fieldDataStorePit = true
grouperDataField.cp_role.fieldDataStorePitDays = 200
grouperDataField.cp_role.fieldDataStructure = attribute
grouperDataField.cp_role.fieldDataType = string
grouperDataField.cp_role.fieldDataUse = access
grouperDataField.cp_role.fieldMultiValued = true
grouperDataField.cp_role.fieldPrivacyRealm = crashplan
grouperDataField.cp_user_id.fieldAliases = cp_user_id
grouperDataField.cp_user_id.fieldDataAssignableTo = individuals
grouperDataField.cp_user_id.fieldDataStorePit = true
grouperDataField.cp_user_id.fieldDataStorePitDays = 200
grouperDataField.cp_user_id.fieldDataStructure = rowColumn
grouperDataField.cp_user_id.fieldDataType = integer
grouperDataField.cp_user_id.fieldDataUse = informational
grouperDataField.cp_user_id.fieldPrivacyRealm = crashplan

grouperDataRow.cp_user.rowAliases = cp_user
grouperDataRow.cp_user.rowDataField.0.colDataFieldConfigId = cp_user_id
grouperDataRow.cp_user.rowDataField.0.rowKeyField = true
grouperDataRow.cp_user.rowDataField.1.colDataFieldConfigId = cp_active
grouperDataRow.cp_user.rowDataField.2.colDataFieldConfigId = cp_blocked
grouperDataRow.cp_user.rowDataField.3.colDataFieldConfigId = cp_known
grouperDataRow.cp_user.rowDataField.4.colDataFieldConfigId = cp_org
grouperDataRow.cp_user.rowNumberOfDataFields = 5
grouperDataRow.cp_user.rowPrivacyRealm = crashplan

grouperDataProviderQuery.crashplan_role.providerConfigId = crashplan
grouperDataProviderQuery.crashplan_role.providerQueryDataField.0.providerDataFieldAttribute = cp_role
grouperDataProviderQuery.crashplan_role.providerQueryDataField.0.providerDataFieldConfigId = cp_role
grouperDataProviderQuery.crashplan_role.providerQueryDataField.0.providerDataFieldMappingType = attribute
grouperDataProviderQuery.crashplan_role.providerQueryDataStructure = attribute
grouperDataProviderQuery.crashplan_role.providerQueryNumberOfDataFields = 1
grouperDataProviderQuery.crashplan_role.providerQuerySqlConfigId = grouper
grouperDataProviderQuery.crashplan_role.providerQuerySqlQuery = select subject_id, cp_role from penn_crashplan_roles_v
grouperDataProviderQuery.crashplan_role.providerQuerySubjectIdAttribute = subject_id
grouperDataProviderQuery.crashplan_role.providerQuerySubjectIdType = subjectId
grouperDataProviderQuery.crashplan_role.providerQueryType = sql
grouperDataProviderQuery.crashplan_user.providerConfigId = crashplan
grouperDataProviderQuery.crashplan_user.providerQueryDataField.0.providerDataFieldAttribute = cp_user_id
grouperDataProviderQuery.crashplan_user.providerQueryDataField.0.providerDataFieldConfigId = cp_user_id
grouperDataProviderQuery.crashplan_user.providerQueryDataField.0.providerDataFieldMappingType = attribute
grouperDataProviderQuery.crashplan_user.providerQueryDataField.1.providerDataFieldAttribute = cp_active
grouperDataProviderQuery.crashplan_user.providerQueryDataField.1.providerDataFieldConfigId = cp_active
grouperDataProviderQuery.crashplan_user.providerQueryDataField.1.providerDataFieldMappingType = attribute
grouperDataProviderQuery.crashplan_user.providerQueryDataField.2.providerDataFieldAttribute = cp_blocked
grouperDataProviderQuery.crashplan_user.providerQueryDataField.2.providerDataFieldConfigId = cp_blocked
grouperDataProviderQuery.crashplan_user.providerQueryDataField.2.providerDataFieldMappingType = attribute
grouperDataProviderQuery.crashplan_user.providerQueryDataField.3.providerDataFieldAttribute = cp_known
grouperDataProviderQuery.crashplan_user.providerQueryDataField.3.providerDataFieldConfigId = cp_known
grouperDataProviderQuery.crashplan_user.providerQueryDataField.3.providerDataFieldMappingType = attribute
grouperDataProviderQuery.crashplan_user.providerQueryDataField.4.providerDataFieldAttribute = cp_org
grouperDataProviderQuery.crashplan_user.providerQueryDataField.4.providerDataFieldConfigId = cp_org
grouperDataProviderQuery.crashplan_user.providerQueryDataField.4.providerDataFieldMappingType = attribute
grouperDataProviderQuery.crashplan_user.providerQueryDataStructure = row
grouperDataProviderQuery.crashplan_user.providerQueryNumberOfDataFields = 5
grouperDataProviderQuery.crashplan_user.providerQueryRowConfigId = cp_user
grouperDataProviderQuery.crashplan_user.providerQuerySqlConfigId = grouper
grouperDataProviderQuery.crashplan_user.providerQuerySqlQuery = select subject_id, cp_user_id, cp_active, cp_blocked, cp_known, cp_org from penn_crashplan_user_v
grouperDataProviderQuery.crashplan_user.providerQuerySubjectIdAttribute = subject_id
grouperDataProviderQuery.crashplan_user.providerQuerySubjectIdType = subjectId
grouperDataProviderQuery.crashplan_user.providerQueryType = sql