diff --git a/spec/tasks/change-labels-task-spec.coffee b/spec/tasks/change-labels-task-spec.coffee
index 22eda8ec0..58b01acdc 100644
--- a/spec/tasks/change-labels-task-spec.coffee
+++ b/spec/tasks/change-labels-task-spec.coffee
@@ -145,12 +145,12 @@ describe "ChangeLabelsTask", ->
           out = task.changesToModel(testThreads['t3'])
           expect(out).toEqual(labels: [testLabels['l3']])
 
-        it 'properly adds and removes labels', ->
+        it 'properly adds and removes labels, ignoring labels that are both added and removed', ->
           task = new ChangeLabelsTask
             labelsToAdd: [testLabels['l1'], testLabels['l2']]
             labelsToRemove: [testLabels['l2'], testLabels['l3']]
           out = task.changesToModel(testThreads['t1'])
-          expect(out).toEqual(labels: [testLabels['l1']])
+          expect(out).toEqual(labels: [testLabels['l1'], testLabels['l2']])
 
         it 'should return an == array of labels when no changes have occurred', ->
           thread = new Thread(id: '1', categories: [testLabels['l2'], testLabels['l3'], testLabels['l1']])
diff --git a/src/flux/tasks/change-labels-task.coffee b/src/flux/tasks/change-labels-task.coffee
index 828d5b74e..7cc5e0715 100644
--- a/src/flux/tasks/change-labels-task.coffee
+++ b/src/flux/tasks/change-labels-task.coffee
@@ -75,8 +75,8 @@ class ChangeLabelsTask extends ChangeMailTask
   changesToModel: (model) ->
     labelsToRemoveIds = _.pluck(@labelsToRemove, 'id')
 
-    labels = [].concat(model.labels, @labelsToAdd)
-    labels = _.reject labels, (label) -> label.id in labelsToRemoveIds
+    labels = _.reject model.labels, (label) -> label.id in labelsToRemoveIds
+    labels = labels.concat(@labelsToAdd)
     labels = _.uniq labels, false, (label) -> label.id
     {labels}