diff --git a/src/postorius/urls.py b/src/postorius/urls.py
index 14fbbc1..6a5c502 100644
--- a/src/postorius/urls.py
+++ b/src/postorius/urls.py
@@ -66,7 +66,7 @@
                                 url(r'^held_messages$',
                                     'list_held_messages',
                                     name='list_held_messages'),
-                                url(r'^(?P<role>[^/]+)/(?P<address>[^/]+)',
+                                url(r'^remove/(?P<role>[^/]+)/(?P<address>[^/]+)',
                                     'remove_role', name='remove_role'),
                                 url(r'^settings/(?P<visible_section>[^/]+)?'
                                     '(?:/(?P<visible_option>.*))?$',