diff --git a/libavutil/Makefile b/libavutil/Makefile
index a73fb79d0f..820abb1429 100644
--- a/libavutil/Makefile
+++ b/libavutil/Makefile
@@ -72,9 +72,6 @@ OBJS = adler32.o                                                        \
        tree.o                                                           \
        utils.o                                                          \
 
-OBJS-$(ARCH_PPC) += ppc/cpu.o
-OBJS-$(ARCH_X86) += x86/cpu.o
-
 TESTPROGS = adler32 aes avstring base64 cpu crc des eval fifo lfg lls \
             md5 opt parseutils sha tree
 
diff --git a/libavutil/ppc/Makefile b/libavutil/ppc/Makefile
new file mode 100644
index 0000000000..5b18b08957
--- /dev/null
+++ b/libavutil/ppc/Makefile
@@ -0,0 +1 @@
+OBJS += ppc/cpu.o                                                       \
diff --git a/libavutil/x86/Makefile b/libavutil/x86/Makefile
new file mode 100644
index 0000000000..de8a341442
--- /dev/null
+++ b/libavutil/x86/Makefile
@@ -0,0 +1 @@
+OBJS += x86/cpu.o                                                       \
diff --git a/libswscale/Makefile b/libswscale/Makefile
index 94c6efb9ba..29f3f12793 100644
--- a/libswscale/Makefile
+++ b/libswscale/Makefile
@@ -12,20 +12,4 @@ OBJS = input.o                                          \
        utils.o                                          \
        yuv2rgb.o                                        \
 
-OBJS-$(ARCH_BFIN)          +=  bfin/internal_bfin.o     \
-                               bfin/swscale_bfin.o      \
-                               bfin/yuv2rgb_bfin.o
-ALTIVEC-OBJS               +=  ppc/swscale_altivec.o    \
-                               ppc/yuv2rgb_altivec.o    \
-                               ppc/yuv2yuv_altivec.o
-MMX-OBJS                   +=  x86/rgb2rgb.o            \
-                               x86/swscale_mmx.o        \
-                               x86/yuv2rgb_mmx.o
-VIS-OBJS                   +=  sparc/yuv2rgb_vis.o
-YASM-OBJS                  +=  x86/input.o              \
-                               x86/output.o             \
-                               x86/scale.o
-
-OBJS-$(CONFIG_XMM_CLOBBER_TEST) += x86/w64xmmtest.o
-
 TESTPROGS = colorspace swscale
diff --git a/libswscale/bfin/Makefile b/libswscale/bfin/Makefile
new file mode 100644
index 0000000000..5f34550427
--- /dev/null
+++ b/libswscale/bfin/Makefile
@@ -0,0 +1,3 @@
+OBJS +=  bfin/internal_bfin.o                                           \
+         bfin/swscale_bfin.o                                            \
+         bfin/yuv2rgb_bfin.o                                            \
diff --git a/libswscale/ppc/Makefile b/libswscale/ppc/Makefile
new file mode 100644
index 0000000000..018955b122
--- /dev/null
+++ b/libswscale/ppc/Makefile
@@ -0,0 +1,3 @@
+ALTIVEC-OBJS +=  ppc/swscale_altivec.o                                  \
+                 ppc/yuv2rgb_altivec.o                                  \
+                 ppc/yuv2yuv_altivec.o                                  \
diff --git a/libswscale/sparc/Makefile b/libswscale/sparc/Makefile
new file mode 100644
index 0000000000..2351ba45ca
--- /dev/null
+++ b/libswscale/sparc/Makefile
@@ -0,0 +1 @@
+VIS-OBJS += sparc/yuv2rgb_vis.o                                         \
diff --git a/libswscale/x86/Makefile b/libswscale/x86/Makefile
new file mode 100644
index 0000000000..7f3779983d
--- /dev/null
+++ b/libswscale/x86/Makefile
@@ -0,0 +1,9 @@
+OBJS-$(CONFIG_XMM_CLOBBER_TEST) += x86/w64xmmtest.o
+
+MMX-OBJS                        += x86/rgb2rgb.o                        \
+                                   x86/swscale_mmx.o                    \
+                                   x86/yuv2rgb_mmx.o                    \
+
+YASM-OBJS                       += x86/input.o                          \
+                                   x86/output.o                         \
+                                   x86/scale.o                          \