diff --git a/src/commands/set-accent.ts b/src/commands/set-accent.ts
index d56d94e..680e518 100644
--- a/src/commands/set-accent.ts
+++ b/src/commands/set-accent.ts
@@ -38,17 +38,30 @@ const quickPick = async (): Promise<string> => {
 };
 
 export const command = async (): Promise<void> => {
+  const pkg = extensionManager.getPackageJSON();
+  const currentThemeID = workspace.getConfiguration().get<string>('workbench.colorTheme');
+  const isMaterialTheme = Boolean(pkg.contributes.themes.find(theme => theme.label === currentThemeID));
+
+  if (!isMaterialTheme) {
+    return;
+  }
+
   const themeConfig = extensionManager.getConfig();
   const currentColorCustomizationsConfig: any = workspace.getConfiguration().get('workbench.colorCustomizations');
   const accent = await quickPick();
+  let config = {};
 
-  const config = accent === PURGE_KEY ? {
-    ...currentColorCustomizationsConfig,
-    ...getThemeColorCustomizationsConfig()
-  } : {
-    ...currentColorCustomizationsConfig,
-    ...getThemeColorCustomizationsConfig(themeConfig.accents[accent])
-  };
+  if (accent === PURGE_KEY) {
+    const {[currentThemeID]: _, ...rest} = currentColorCustomizationsConfig;
+    config = rest;
+  } else {
+    config = {
+      ...currentColorCustomizationsConfig,
+      [`[${currentThemeID}]`]: {
+        ...getThemeColorCustomizationsConfig(themeConfig.accents[accent])
+      }
+    };
+  }
 
   await updateColorCustomizationsConfig(config);
   await settingsManager.updateSetting('accent', accent);
diff --git a/src/core/extension-manager.ts b/src/core/extension-manager.ts
index 0b2a9bd..2db6a65 100644
--- a/src/core/extension-manager.ts
+++ b/src/core/extension-manager.ts
@@ -8,6 +8,15 @@ type MaterialThemeConfig = {
   changelog?: { lastversion?: string };
 };
 
+type PackageJSON = {
+  version: string;
+  contributes: {
+    themes: Array<{
+      label: string;
+    }>;
+  };
+};
+
 type InstallationType = {
   firstInstall: boolean;
   update: boolean;
@@ -15,7 +24,7 @@ type InstallationType = {
 
 export interface IExtensionManager {
   init: () => Promise<void>;
-  getPackageJSON: () => Record<string, any>;
+  getPackageJSON: () => PackageJSON;
   getConfig: () => MaterialThemeConfig;
   getInstallationType: () => Record<string, unknown>;
   updateConfig: (config: Partial<MaterialThemeConfig>) => Promise<void>;
@@ -33,7 +42,7 @@ class ExtensionManager implements IExtensionManager {
     this.userConfigFileUri = extensionFolderUri.with({path: posix.join(extensionFolderUri.path, USER_CONFIG_FILE_NAME)});
   }
 
-  getPackageJSON(): Record<string, any> {
+  getPackageJSON(): PackageJSON {
     return extensions.getExtension(MATERIAL_THEME_EXT_ID).packageJSON;
   }