diff --git a/lib/elf.c b/lib/elf.c
index dc13935e103..28ec87b8e48 100644
--- a/lib/elf.c
+++ b/lib/elf.c
@@ -90,6 +90,10 @@ unsigned long load_elf64_image_phdr(unsigned long addr)
 		void *dst = (void *)(ulong)phdr->p_paddr;
 		void *src = (void *)addr + phdr->p_offset;
 
+		/* Only load PT_LOAD program header */
+		if (phdr->p_type != PT_LOAD)
+			continue;
+
 		debug("Loading phdr %i to 0x%p (%lu bytes)\n",
 		      i, dst, (ulong)phdr->p_filesz);
 		if (phdr->p_filesz)
@@ -205,6 +209,10 @@ unsigned long load_elf_image_phdr(unsigned long addr)
 		void *dst = (void *)(uintptr_t)phdr->p_paddr;
 		void *src = (void *)addr + phdr->p_offset;
 
+		/* Only load PT_LOAD program header */
+		if (phdr->p_type != PT_LOAD)
+			continue;
+
 		debug("Loading phdr %i to 0x%p (%i bytes)\n",
 		      i, dst, phdr->p_filesz);
 		if (phdr->p_filesz)