diff --git a/CHANGELOG b/CHANGELOG
index 173886159080cd2f6aea68b07d529d1c061ee06a..8e85ef8bbed4bc4eb24ed806325b241cde7dc70f 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -37,6 +37,7 @@ v 8.12.0 (unreleased)
   - Use 'git update-ref' for safer web commits !6130
   - Automatically expand hidden discussions when accessed by a permalink !5585 (Mike Greiling)
   - Remove unused mixins (ClemMakesApps)
+  - Fix issue board label filtering appending already filtered labels
   - Add search to all issue board lists
   - Fix groups sort dropdown alignment (ClemMakesApps)
   - Add horizontal scrolling to all sub-navs on mobile viewports (ClemMakesApps)
diff --git a/app/assets/javascripts/labels_select.js b/app/assets/javascripts/labels_select.js
index bab23ff5ac04ffbc32d5093f0584b8eaa48913a7..5f84e0f102462208bcc119c2b13ea6dbc3bfaeec 100644
--- a/app/assets/javascripts/labels_select.js
+++ b/app/assets/javascripts/labels_select.js
@@ -273,12 +273,12 @@
             if (page === 'projects:boards:show') {
               if (label.isAny) {
                 gl.issueBoards.BoardsStore.state.filters['label_name'] = [];
-              } else if (label.title) {
+              } else if ($el.hasClass('is-active')) {
                 gl.issueBoards.BoardsStore.state.filters['label_name'].push(label.title);
               } else {
                 var filters = gl.issueBoards.BoardsStore.state.filters['label_name'];
-                filters = filters.filter(function (label) {
-                  return label !== $el.text().trim();
+                filters = filters.filter(function (filteredLabel) {
+                  return filteredLabel !== label.title;
                 });
                 gl.issueBoards.BoardsStore.state.filters['label_name'] = filters;
               }
diff --git a/spec/features/boards/boards_spec.rb b/spec/features/boards/boards_spec.rb
index c6c2e2095dfdc93694b103f46ab4a56d53171ddd..c697dbea131f2b25a5d96815b1bc8c75160271f5 100644
--- a/spec/features/boards/boards_spec.rb
+++ b/spec/features/boards/boards_spec.rb
@@ -467,6 +467,25 @@ describe 'Issue Boards', feature: true, js: true do
         end
       end
 
+      it 'removes filtered labels' do
+        page.within '.labels-filter' do
+          click_button('Label')
+          wait_for_ajax
+
+          page.within '.dropdown-menu-labels' do
+            click_link(testing.title)
+          end
+
+          expect(page).to have_css('input[name="label_name[]"]', visible: false)
+
+          page.within '.dropdown-menu-labels' do
+            click_link(testing.title)
+          end
+
+          expect(page).not_to have_css('input[name="label_name[]"]', visible: false)
+        end
+      end
+
       it 'infinite scrolls list with label filter' do
         50.times do
           create(:labeled_issue, project: project, labels: [testing])