diff --git a/CHANGELOG b/CHANGELOG
index 2f3fe4703d4b3c1ab07c68e72f98a10196a36bc2..28c5d23d604c2d8770a96402755ffe6567cd9ec4 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -118,6 +118,7 @@ v 8.12.0
   - Change pipeline duration to be jobs running time instead of simple wall time from start to end !6084
   - Show queued time when showing a pipeline !6084
   - Remove unused mixins (ClemMakesApps)
+  - Fix issue board label filtering appending already filtered labels
   - Add search to all issue board lists
   - Scroll active tab into view on mobile
   - Fix groups sort dropdown alignment (ClemMakesApps)
diff --git a/app/assets/javascripts/labels_select.js b/app/assets/javascripts/labels_select.js
index 3f15a117ca887bfdb2e250cfb2dfd081a7759d51..c562b7d7b00a63c3b9e1ac38c696f4c6e420d6c9 100644
--- a/app/assets/javascripts/labels_select.js
+++ b/app/assets/javascripts/labels_select.js
@@ -280,12 +280,12 @@
             if (page === 'projects:boards:show') {
               if (label.isAny) {
                 gl.issueBoards.BoardsStore.state.filters['label_name'] = [];
-              } else if (label.title) {
+              } else if ($el.hasClass('is-active')) {
                 gl.issueBoards.BoardsStore.state.filters['label_name'].push(label.title);
               } else {
                 var filters = gl.issueBoards.BoardsStore.state.filters['label_name'];
-                filters = filters.filter(function (label) {
-                  return label !== $el.text().trim();
+                filters = filters.filter(function (filteredLabel) {
+                  return filteredLabel !== label.title;
                 });
                 gl.issueBoards.BoardsStore.state.filters['label_name'] = filters;
               }
diff --git a/spec/features/boards/boards_spec.rb b/spec/features/boards/boards_spec.rb
index 2dcbc4703e146eec7aed2edd220593be4393dc06..26ea06e002b1c82be72d8c619a0486ac6640f1b6 100644
--- a/spec/features/boards/boards_spec.rb
+++ b/spec/features/boards/boards_spec.rb
@@ -466,6 +466,29 @@ describe 'Issue Boards', feature: true, js: true do
         end
       end
 
+      it 'removes filtered labels' do
+        wait_for_vue_resource
+        
+        page.within '.labels-filter' do
+          click_button('Label')
+          wait_for_ajax
+
+          page.within '.dropdown-menu-labels' do
+            click_link(testing.title)
+            wait_for_vue_resource(spinner: false)
+          end
+
+          expect(page).to have_css('input[name="label_name[]"]', visible: false)
+
+          page.within '.dropdown-menu-labels' do
+            click_link(testing.title)
+            wait_for_vue_resource(spinner: false)
+          end
+
+          expect(page).not_to have_css('input[name="label_name[]"]', visible: false)
+        end
+      end
+
       it 'infinite scrolls list with label filter' do
         50.times do
           create(:labeled_issue, project: project, labels: [testing])